diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index a6a37f8bec332..bfa783ff01160 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1499,12 +1499,6 @@ public Builder put(String name, ComponentTemplate componentTemplate) { return this; } - @Deprecated(forRemoval = true) - public Builder removeComponentTemplate(String name) { - getSingleProject().removeComponentTemplate(name); - return this; - } - @Deprecated(forRemoval = true) public Builder componentTemplates(Map componentTemplates) { getSingleProject().componentTemplates(componentTemplates); @@ -1523,12 +1517,6 @@ public Builder put(String name, ComposableIndexTemplate indexTemplate) { return this; } - @Deprecated(forRemoval = true) - public Builder removeIndexTemplate(String name) { - getSingleProject().removeIndexTemplate(name); - return this; - } - @Deprecated(forRemoval = true) public Builder dataStreams(Map dataStreams, Map dataStreamAliases) { getSingleProject().dataStreams(dataStreams, dataStreamAliases); @@ -1557,11 +1545,6 @@ public Builder removeDataStream(String name) { return this; } - @Deprecated(forRemoval = true) - public boolean removeDataStreamAlias(String aliasName, String dataStreamName, boolean mustExist) { - return getSingleProject().removeDataStreamAlias(aliasName, dataStreamName, mustExist); - } - public Builder putCustom(String type, ClusterCustom custom) { customs.put(type, Objects.requireNonNull(custom, type)); return this; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 6eecf4921c50e..01e271ff6483b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -9,11 +9,8 @@ package org.elasticsearch.cluster.metadata; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.cluster.ClusterModule; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.coordination.CoordinationMetadata; import org.elasticsearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; @@ -22,7 +19,6 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -33,22 +29,13 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.FixForMultiProject; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Predicates; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.health.node.selection.HealthNode; import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor; import org.elasticsearch.health.node.selection.HealthNodeTaskParams; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.alias.RandomAliasActionsGenerator; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.persistent.ClusterPersistentTasksCustomMetadata; @@ -56,8 +43,6 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksExecutorRegistry; import org.elasticsearch.persistent.PersistentTasksService; -import org.elasticsearch.plugins.FieldPredicate; -import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; @@ -89,31 +74,21 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; -import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; -import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_API; import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_SNAPSHOT; -import static org.elasticsearch.cluster.metadata.ProjectMetadata.Builder.assertDataStreams; -import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty; import static org.elasticsearch.test.hamcrest.OptionalMatchers.isPresentWith; import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -121,472 +96,11 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.hamcrest.Matchers.startsWith; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class MetadataTests extends ESTestCase { - public void testFindAliases() { - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("alias1").build()) - .putAlias(AliasMetadata.builder("alias2").build()) - ) - .put( - IndexMetadata.builder("index2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("alias2").build()) - .putAlias(AliasMetadata.builder("alias3").build()) - ) - .build(); - - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT); - Map> aliases = metadata.getProject().findAliases(request.aliases(), Strings.EMPTY_ARRAY); - assertThat(aliases, anEmptyMap()); - } - { - final GetAliasesRequest request; - if (randomBoolean()) { - request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT); - } else { - request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, randomFrom("alias1", "alias2")); - // replacing with empty aliases behaves as if aliases were unspecified at request building - request.replaceAliases(Strings.EMPTY_ARRAY); - } - Map> aliases = metadata.getProject().findAliases(request.aliases(), new String[] { "index" }); - assertThat(aliases, aMapWithSize(1)); - List aliasMetadataList = aliases.get("index"); - assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2"))); - } - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias*"); - Map> aliases = metadata.getProject() - .findAliases(request.aliases(), new String[] { "index", "index2" }); - assertThat(aliases, aMapWithSize(2)); - List indexAliasMetadataList = aliases.get("index"); - assertThat(indexAliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2"))); - List index2AliasMetadataList = aliases.get("index2"); - assertThat(index2AliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias2", "alias3"))); - } - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias1"); - Map> aliases = metadata.getProject().findAliases(request.aliases(), new String[] { "index" }); - assertThat(aliases, aMapWithSize(1)); - List aliasMetadataList = aliases.get("index"); - assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1"))); - } - { - Map> aliases = metadata.getProject().findAllAliases(new String[] { "index" }); - assertThat(aliases, aMapWithSize(1)); - List aliasMetadataList = aliases.get("index"); - assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2"))); - } - { - Map> aliases = metadata.getProject().findAllAliases(Strings.EMPTY_ARRAY); - assertThat(aliases, anEmptyMap()); - } - } - - public void testFindDataStreamAliases() { - Metadata.Builder builder = Metadata.builder(); - - addDataStream("d1", builder); - addDataStream("d2", builder); - addDataStream("d3", builder); - addDataStream("d4", builder); - - builder.put("alias1", "d1", null, null); - builder.put("alias2", "d2", null, null); - builder.put("alias2-part2", "d2", null, null); - - Metadata metadata = builder.build(); - - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT); - Map> aliases = metadata.getProject() - .findDataStreamAliases(request.aliases(), Strings.EMPTY_ARRAY); - assertThat(aliases, anEmptyMap()); - } - - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("alias1"); - Map> aliases = metadata.getProject() - .findDataStreamAliases(request.aliases(), new String[] { "index" }); - assertThat(aliases, anEmptyMap()); - } - - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("alias1"); - Map> aliases = metadata.getProject() - .findDataStreamAliases(request.aliases(), new String[] { "index", "d1", "d2" }); - assertEquals(1, aliases.size()); - List found = aliases.get("d1"); - assertThat(found, transformedItemsMatch(DataStreamAlias::getAlias, contains("alias1"))); - } - - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("ali*"); - Map> aliases = metadata.getProject() - .findDataStreamAliases(request.aliases(), new String[] { "index", "d2" }); - assertEquals(1, aliases.size()); - List found = aliases.get("d2"); - assertThat(found, transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); - } - - // test exclusion - { - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("*"); - Map> aliases = metadata.getProject() - .findDataStreamAliases(request.aliases(), new String[] { "index", "d1", "d2", "d3", "d4" }); - assertThat(aliases.get("d2"), transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); - assertThat(aliases.get("d1"), transformedItemsMatch(DataStreamAlias::getAlias, contains("alias1"))); - - request.aliases("*", "-alias1"); - aliases = metadata.getProject().findDataStreamAliases(request.aliases(), new String[] { "index", "d1", "d2", "d3", "d4" }); - assertThat(aliases.get("d2"), transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); - assertNull(aliases.get("d1")); - } - } - - public void testDataStreamAliasesByDataStream() { - Metadata.Builder builder = Metadata.builder(); - - addDataStream("d1", builder); - addDataStream("d2", builder); - addDataStream("d3", builder); - addDataStream("d4", builder); - - builder.put("alias1", "d1", null, null); - builder.put("alias2", "d2", null, null); - builder.put("alias2-part2", "d2", null, null); - - Metadata metadata = builder.build(); - - var aliases = metadata.getProject().dataStreamAliasesByDataStream(); - - assertTrue(aliases.containsKey("d1")); - assertTrue(aliases.containsKey("d2")); - assertFalse(aliases.containsKey("d3")); - assertFalse(aliases.containsKey("d4")); - - assertEquals(1, aliases.get("d1").size()); - assertEquals(2, aliases.get("d2").size()); - - assertThat(aliases.get("d2"), transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); - } - - public void testFindAliasWithExclusion() { - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("alias1").build()) - .putAlias(AliasMetadata.builder("alias2").build()) - ) - .put( - IndexMetadata.builder("index2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("alias1").build()) - .putAlias(AliasMetadata.builder("alias3").build()) - ) - .build(); - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("*", "-alias1"); - Map> aliases = metadata.getProject().findAliases(request.aliases(), new String[] { "index", "index2" }); - assertThat(aliases.get("index"), transformedItemsMatch(AliasMetadata::alias, contains("alias2"))); - assertThat(aliases.get("index2"), transformedItemsMatch(AliasMetadata::alias, contains("alias3"))); - } - - public void testFindDataStreams() { - final int numIndices = randomIntBetween(2, 5); - final int numBackingIndices = randomIntBetween(2, 5); - final String dataStreamName = "my-data-stream"; - CreateIndexResult result = createIndices(numIndices, numBackingIndices, dataStreamName); - - List allIndices = new ArrayList<>(result.indices); - allIndices.addAll(result.backingIndices); - String[] concreteIndices = allIndices.stream().map(Index::getName).toArray(String[]::new); - Map dataStreams = result.metadata.getProject().findDataStreams(concreteIndices); - assertThat(dataStreams, aMapWithSize(numBackingIndices)); - for (Index backingIndex : result.backingIndices) { - assertThat(dataStreams, hasKey(backingIndex.getName())); - assertThat(dataStreams.get(backingIndex.getName()).getName(), equalTo(dataStreamName)); - } - } - - public void testFindAliasWithExclusionAndOverride() { - Metadata metadata = Metadata.builder() - .put( - IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("aa").build()) - .putAlias(AliasMetadata.builder("ab").build()) - .putAlias(AliasMetadata.builder("bb").build()) - ) - .build(); - GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("a*", "-*b", "b*"); - List aliases = metadata.getProject().findAliases(request.aliases(), new String[] { "index" }).get("index"); - assertThat(aliases, transformedItemsMatch(AliasMetadata::alias, contains("aa", "bb"))); - } - - public void testAliasCollidingWithAnExistingIndex() { - int indexCount = randomIntBetween(10, 100); - Set indices = Sets.newHashSetWithExpectedSize(indexCount); - for (int i = 0; i < indexCount; i++) { - indices.add(randomAlphaOfLength(10)); - } - Map> aliasToIndices = new HashMap<>(); - for (String alias : randomSubsetOf(randomIntBetween(1, 10), indices)) { - Set indicesInAlias; - do { - indicesInAlias = new HashSet<>(randomSubsetOf(randomIntBetween(1, 3), indices)); - indicesInAlias.remove(alias); - } while (indicesInAlias.isEmpty()); - aliasToIndices.put(alias, indicesInAlias); - } - int properAliases = randomIntBetween(0, 3); - for (int i = 0; i < properAliases; i++) { - aliasToIndices.put(randomAlphaOfLength(5), new HashSet<>(randomSubsetOf(randomIntBetween(1, 3), indices))); - } - Metadata.Builder metadataBuilder = Metadata.builder(); - for (String index : indices) { - IndexMetadata.Builder indexBuilder = IndexMetadata.builder(index) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0); - aliasToIndices.forEach((key, value) -> { - if (value.contains(index)) { - indexBuilder.putAlias(AliasMetadata.builder(key).build()); - } - }); - metadataBuilder.put(indexBuilder); - } - - Exception e = expectThrows(IllegalStateException.class, metadataBuilder::build); - assertThat(e.getMessage(), startsWith("index, alias, and data stream names need to be unique")); - } - - public void testValidateAliasWriteOnly() { - String alias = randomAlphaOfLength(5); - String indexA = randomAlphaOfLength(6); - String indexB = randomAlphaOfLength(7); - Boolean aWriteIndex = randomBoolean() ? null : randomBoolean(); - Boolean bWriteIndex; - if (Boolean.TRUE.equals(aWriteIndex)) { - bWriteIndex = randomFrom(Boolean.FALSE, null); - } else { - bWriteIndex = randomFrom(Boolean.TRUE, Boolean.FALSE, null); - } - // when only one index/alias pair exist - Metadata metadata = Metadata.builder().put(buildIndexMetadata(indexA, alias, aWriteIndex)).build(); - - // when alias points to two indices, but valid - // one of the following combinations: [(null, null), (null, true), (null, false), (false, false)] - Metadata.builder(metadata).put(buildIndexMetadata(indexB, alias, bWriteIndex)).build(); - - // when too many write indices - Exception exception = expectThrows(IllegalStateException.class, () -> { - IndexMetadata.Builder metaA = buildIndexMetadata(indexA, alias, true); - IndexMetadata.Builder metaB = buildIndexMetadata(indexB, alias, true); - Metadata.builder().put(metaA).put(metaB).build(); - }); - assertThat(exception.getMessage(), startsWith("alias [" + alias + "] has more than one write index [")); - } - - public void testValidateHiddenAliasConsistency() { - String alias = randomAlphaOfLength(5); - String indexA = randomAlphaOfLength(6); - String indexB = randomAlphaOfLength(7); - - { - Exception ex = expectThrows( - IllegalStateException.class, - () -> buildMetadataWithHiddenIndexMix(alias, indexA, true, indexB, randomFrom(false, null)).build() - ); - assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); - } - - { - Exception ex = expectThrows( - IllegalStateException.class, - () -> buildMetadataWithHiddenIndexMix(alias, indexA, randomFrom(false, null), indexB, true).build() - ); - assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); - } - } - - private Metadata.Builder buildMetadataWithHiddenIndexMix( - String aliasName, - String indexAName, - Boolean indexAHidden, - String indexBName, - Boolean indexBHidden - ) { - IndexMetadata.Builder indexAMeta = IndexMetadata.builder(indexAName) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder(aliasName).isHidden(indexAHidden).build()); - IndexMetadata.Builder indexBMeta = IndexMetadata.builder(indexBName) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder(aliasName).isHidden(indexBHidden).build()); - return Metadata.builder().put(indexAMeta).put(indexBMeta); - } - - public void testResolveIndexRouting() { - IndexMetadata.Builder builder = IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("alias0").build()) - .putAlias(AliasMetadata.builder("alias1").routing("1").build()) - .putAlias(AliasMetadata.builder("alias2").routing("1,2").build()); - Metadata metadata = Metadata.builder().put(builder).build(); - - // no alias, no index - assertNull(metadata.getProject().resolveIndexRouting(null, null)); - assertEquals(metadata.getProject().resolveIndexRouting("0", null), "0"); - - // index, no alias - assertNull(metadata.getProject().resolveIndexRouting(null, "index")); - assertEquals(metadata.getProject().resolveIndexRouting("0", "index"), "0"); - - // alias with no index routing - assertNull(metadata.getProject().resolveIndexRouting(null, "alias0")); - assertEquals(metadata.getProject().resolveIndexRouting("0", "alias0"), "0"); - - // alias with index routing. - assertEquals(metadata.getProject().resolveIndexRouting(null, "alias1"), "1"); - Exception ex = expectThrows(IllegalArgumentException.class, () -> metadata.getProject().resolveIndexRouting("0", "alias1")); - assertThat( - ex.getMessage(), - is("Alias [alias1] has index routing associated with it [1], and was provided with routing value [0], rejecting operation") - ); - - // alias with invalid index routing. - ex = expectThrows(IllegalArgumentException.class, () -> metadata.getProject().resolveIndexRouting(null, "alias2")); - assertThat( - ex.getMessage(), - is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") - ); - - ex = expectThrows(IllegalArgumentException.class, () -> metadata.getProject().resolveIndexRouting("1", "alias2")); - assertThat( - ex.getMessage(), - is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") - ); - - IndexMetadata.Builder builder2 = IndexMetadata.builder("index2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(AliasMetadata.builder("alias0").build()); - Metadata metadataTwoIndices = Metadata.builder(metadata).put(builder2).build(); - - // alias with multiple indices - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> metadataTwoIndices.getProject().resolveIndexRouting("1", "alias0") - ); - assertThat(exception.getMessage(), startsWith("Alias [alias0] has more than one index associated with it")); - } - - public void testResolveWriteIndexRouting() { - AliasMetadata.Builder aliasZeroBuilder = AliasMetadata.builder("alias0"); - if (randomBoolean()) { - aliasZeroBuilder.writeIndex(true); - } - IndexMetadata.Builder builder = IndexMetadata.builder("index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(aliasZeroBuilder.build()) - .putAlias(AliasMetadata.builder("alias1").routing("1").build()) - .putAlias(AliasMetadata.builder("alias2").routing("1,2").build()) - .putAlias(AliasMetadata.builder("alias3").writeIndex(false).build()) - .putAlias(AliasMetadata.builder("alias4").routing("1,2").writeIndex(true).build()); - Metadata metadata = Metadata.builder().put(builder).build(); - - // no alias, no index - assertNull(metadata.getProject().resolveWriteIndexRouting(null, null)); - assertEquals(metadata.getProject().resolveWriteIndexRouting("0", null), "0"); - - // index, no alias - assertNull(metadata.getProject().resolveWriteIndexRouting(null, "index")); - assertEquals(metadata.getProject().resolveWriteIndexRouting("0", "index"), "0"); - - // alias with no index routing - assertNull(metadata.getProject().resolveWriteIndexRouting(null, "alias0")); - assertEquals(metadata.getProject().resolveWriteIndexRouting("0", "alias0"), "0"); - - // alias with index routing. - assertEquals(metadata.getProject().resolveWriteIndexRouting(null, "alias1"), "1"); - Exception exception = expectThrows( - IllegalArgumentException.class, - () -> metadata.getProject().resolveWriteIndexRouting("0", "alias1") - ); - assertThat( - exception.getMessage(), - is("Alias [alias1] has index routing associated with it [1], and was provided with routing value [0], rejecting operation") - ); - - // alias with invalid index routing. - exception = expectThrows(IllegalArgumentException.class, () -> metadata.getProject().resolveWriteIndexRouting(null, "alias2")); - assertThat( - exception.getMessage(), - is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") - ); - exception = expectThrows(IllegalArgumentException.class, () -> metadata.getProject().resolveWriteIndexRouting("1", "alias2")); - assertThat( - exception.getMessage(), - is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") - ); - exception = expectThrows( - IllegalArgumentException.class, - () -> metadata.getProject().resolveWriteIndexRouting(randomFrom("1", null), "alias4") - ); - assertThat( - exception.getMessage(), - is("index/alias [alias4] provided with routing value [1,2] that resolved to several routing values, rejecting operation") - ); - - // alias with no write index - exception = expectThrows(IllegalArgumentException.class, () -> metadata.getProject().resolveWriteIndexRouting("1", "alias3")); - assertThat(exception.getMessage(), is("alias [alias3] does not have a write index")); - - // aliases with multiple indices - AliasMetadata.Builder aliasZeroBuilderTwo = AliasMetadata.builder("alias0"); - if (randomBoolean()) { - aliasZeroBuilder.writeIndex(false); - } - IndexMetadata.Builder builder2 = IndexMetadata.builder("index2") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(aliasZeroBuilderTwo.build()) - .putAlias(AliasMetadata.builder("alias1").routing("0").writeIndex(true).build()) - .putAlias(AliasMetadata.builder("alias2").writeIndex(true).build()); - Metadata metadataTwoIndices = Metadata.builder(metadata).put(builder2).build(); - - // verify that new write index is used - assertThat("0", equalTo(metadataTwoIndices.getProject().resolveWriteIndexRouting("0", "alias1"))); - } - public void testUnknownFieldClusterMetadata() throws IOException { BytesReference metadata = BytesReference.bytes( JsonXContent.contentBuilder().startObject().startObject("meta-data").field("random", "value").endObject().endObject() @@ -837,7 +351,7 @@ public void testParseXContentFormatBeforeMultiProject() throws IOException { containsInAnyOrder("health-node") ); assertThat( - metadata.getProject().customs().keySet(), + metadata.getProject(ProjectId.DEFAULT).customs().keySet(), containsInAnyOrder("persistent_tasks", "index-graveyard", "component_template", "repositories") ); assertThat(metadata.customs(), not(hasKey("repositories"))); @@ -915,1719 +429,196 @@ public void testParseXContentFormatBeforeRepositoriesMetadataMigration() throws "location": "backup" }, "generation": 42, - "pending_generation": 42 - } - }, - "reserved_state":{ } - } - } - """, IndexVersion.current(), IndexVersion.current()); - - final Metadata metadata = fromJsonXContentStringWithPersistentTasks(json); - assertThat(metadata, notNullValue()); - assertThat(metadata.clusterUUID(), is("aba1aa1ababbbaabaabaab")); - - assertThat(metadata.projects().keySet(), containsInAnyOrder(ProjectId.fromId("default"), ProjectId.fromId("another_project"))); - assertThat(metadata.customs(), not(hasKey("repositories"))); - final var repositoriesMetadata = RepositoriesMetadata.get(metadata.getProject(ProjectId.DEFAULT)); - assertThat( - repositoriesMetadata.repositories(), - equalTo( - List.of( - new RepositoryMetadata("my-repo", "_my-repo-uuid_", "fs", Settings.builder().put("location", "backup").build(), 42, 42) - ) - ) - ); - assertThat(metadata.getProject(ProjectId.fromId("another_project")).customs(), not(hasKey("repositories"))); - } - - private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws IOException { - List registry = new ArrayList<>(); - registry.addAll(ClusterModule.getNamedXWriteables()); - registry.addAll(IndicesModule.getNamedXContents()); - registry.addAll(HealthNodeTaskExecutor.getNamedXContentParsers()); - - final var clusterService = mock(ClusterService.class); - when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class)); - final var healthNodeTaskExecutor = HealthNodeTaskExecutor.create( - clusterService, - mock(PersistentTasksService.class), - Settings.EMPTY, - ClusterSettings.createBuiltInClusterSettings() - ); - new PersistentTasksExecutorRegistry(List.of(healthNodeTaskExecutor)); - - XContentParserConfiguration config = XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(registry)); - try (XContentParser parser = JsonXContent.jsonXContent.createParser(config, json)) { - return Metadata.fromXContent(parser); - } - } - - public void testGlobalStateEqualsCoordinationMetadata() { - CoordinationMetadata coordinationMetadata1 = new CoordinationMetadata( - randomNonNegativeLong(), - randomVotingConfig(), - randomVotingConfig(), - randomVotingConfigExclusions() - ); - Metadata metadata1 = Metadata.builder().coordinationMetadata(coordinationMetadata1).build(); - CoordinationMetadata coordinationMetadata2 = new CoordinationMetadata( - randomNonNegativeLong(), - randomVotingConfig(), - randomVotingConfig(), - randomVotingConfigExclusions() - ); - Metadata metadata2 = Metadata.builder().coordinationMetadata(coordinationMetadata2).build(); - - assertTrue(Metadata.isGlobalStateEquals(metadata1, metadata1)); - assertFalse(Metadata.isGlobalStateEquals(metadata1, metadata2)); - } - - public void testSerializationWithIndexGraveyard() throws IOException { - final var projectId = randomProjectIdOrDefault(); - final IndexGraveyard graveyard = IndexGraveyardTests.createRandom(); - final Metadata originalMeta = Metadata.builder().put(ProjectMetadata.builder(projectId).indexGraveyard(graveyard)).build(); - final BytesStreamOutput out = new BytesStreamOutput(); - originalMeta.writeTo(out); - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); - final Metadata fromStreamMeta = Metadata.readFrom( - new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) - ); - assertThat(fromStreamMeta.getProject(projectId).indexGraveyard(), equalTo(originalMeta.getProject(projectId).indexGraveyard())); - } - - public void testFindMappings() throws IOException { - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("index1").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(FIND_MAPPINGS_TEST_ITEM)) - .put(IndexMetadata.builder("index2").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(FIND_MAPPINGS_TEST_ITEM)) - .build(); - - { - AtomicInteger onNextIndexCalls = new AtomicInteger(0); - Map mappings = metadata.getProject() - .findMappings(Strings.EMPTY_ARRAY, MapperPlugin.NOOP_FIELD_FILTER, onNextIndexCalls::incrementAndGet); - assertThat(mappings, anEmptyMap()); - assertThat(onNextIndexCalls.get(), equalTo(0)); - } - { - AtomicInteger onNextIndexCalls = new AtomicInteger(0); - Map mappings = metadata.getProject() - .findMappings(new String[] { "index1" }, MapperPlugin.NOOP_FIELD_FILTER, onNextIndexCalls::incrementAndGet); - assertThat(mappings, aMapWithSize(1)); - assertIndexMappingsNotFiltered(mappings, "index1"); - assertThat(onNextIndexCalls.get(), equalTo(1)); - } - { - AtomicInteger onNextIndexCalls = new AtomicInteger(0); - Map mappings = metadata.getProject() - .findMappings(new String[] { "index1", "index2" }, MapperPlugin.NOOP_FIELD_FILTER, onNextIndexCalls::incrementAndGet); - assertThat(mappings, aMapWithSize(2)); - assertIndexMappingsNotFiltered(mappings, "index1"); - assertIndexMappingsNotFiltered(mappings, "index2"); - assertThat(onNextIndexCalls.get(), equalTo(2)); - } - } - - public void testFindMappingsNoOpFilters() throws IOException { - MappingMetadata originalMappingMetadata = new MappingMetadata( - "_doc", - XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, true) - ); - - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("index1").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(originalMappingMetadata)) - .build(); - - { - Map mappings = metadata.getProject() - .findMappings(new String[] { "index1" }, MapperPlugin.NOOP_FIELD_FILTER, Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP); - MappingMetadata mappingMetadata = mappings.get("index1"); - assertSame(originalMappingMetadata, mappingMetadata); - } - { - Map mappings = metadata.getProject() - .findMappings(new String[] { "index1" }, index -> field -> randomBoolean(), Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP); - MappingMetadata mappingMetadata = mappings.get("index1"); - assertNotSame(originalMappingMetadata, mappingMetadata); - } - } - - @SuppressWarnings("unchecked") - public void testFindMappingsWithFilters() throws IOException { - String mapping = FIND_MAPPINGS_TEST_ITEM; - if (randomBoolean()) { - Map stringObjectMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, false); - Map doc = (Map) stringObjectMap.get("_doc"); - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.map(doc); - mapping = Strings.toString(builder); - } - } - - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("index1").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(mapping)) - .put(IndexMetadata.builder("index2").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(mapping)) - .put(IndexMetadata.builder("index3").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(mapping)) - .build(); - - { - Map mappings = metadata.getProject() - .findMappings(new String[] { "index1", "index2", "index3" }, index -> { - if (index.equals("index1")) { - return field -> field.startsWith("name.") == false - && field.startsWith("properties.key.") == false - && field.equals("age") == false - && field.equals("address.location") == false; - } - if (index.equals("index2")) { - return Predicates.never(); - } - return FieldPredicate.ACCEPT_ALL; - }, Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP); - - assertIndexMappingsNoFields(mappings, "index2"); - assertIndexMappingsNotFiltered(mappings, "index3"); - - MappingMetadata docMapping = mappings.get("index1"); - assertNotNull(docMapping); - - Map sourceAsMap = docMapping.getSourceAsMap(); - assertThat(sourceAsMap.keySet(), containsInAnyOrder("properties", "_routing", "_source")); - - Map typeProperties = (Map) sourceAsMap.get("properties"); - assertThat(typeProperties.keySet(), containsInAnyOrder("name", "address", "birth", "ip", "suggest", "properties")); - - Map name = (Map) typeProperties.get("name"); - assertThat(name.keySet(), containsInAnyOrder("properties")); - Map nameProperties = (Map) name.get("properties"); - assertThat(nameProperties, anEmptyMap()); - - Map address = (Map) typeProperties.get("address"); - assertThat(address.keySet(), containsInAnyOrder("type", "properties")); - Map addressProperties = (Map) address.get("properties"); - assertThat(addressProperties.keySet(), containsInAnyOrder("street", "area")); - assertLeafs(addressProperties, "street", "area"); - - Map properties = (Map) typeProperties.get("properties"); - assertThat(properties.keySet(), containsInAnyOrder("type", "properties")); - Map propertiesProperties = (Map) properties.get("properties"); - assertThat(propertiesProperties.keySet(), containsInAnyOrder("key", "value")); - assertLeafs(propertiesProperties, "key"); - assertMultiField(propertiesProperties, "value", "keyword"); - } - - { - Map mappings = metadata.getProject() - .findMappings( - new String[] { "index1", "index2", "index3" }, - index -> field -> (index.equals("index3") && field.endsWith("keyword")), - Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP - ); - - assertIndexMappingsNoFields(mappings, "index1"); - assertIndexMappingsNoFields(mappings, "index2"); - MappingMetadata mappingMetadata = mappings.get("index3"); - Map sourceAsMap = mappingMetadata.getSourceAsMap(); - assertThat(sourceAsMap.keySet(), containsInAnyOrder("_routing", "_source", "properties")); - Map typeProperties = (Map) sourceAsMap.get("properties"); - assertThat(typeProperties.keySet(), containsInAnyOrder("properties")); - Map properties = (Map) typeProperties.get("properties"); - assertThat(properties.keySet(), containsInAnyOrder("type", "properties")); - Map propertiesProperties = (Map) properties.get("properties"); - assertThat(propertiesProperties.keySet(), containsInAnyOrder("key", "value")); - Map key = (Map) propertiesProperties.get("key"); - assertThat(key.keySet(), containsInAnyOrder("properties")); - Map keyProperties = (Map) key.get("properties"); - assertThat(keyProperties.keySet(), containsInAnyOrder("keyword")); - assertLeafs(keyProperties, "keyword"); - Map value = (Map) propertiesProperties.get("value"); - assertThat(value.keySet(), containsInAnyOrder("properties")); - Map valueProperties = (Map) value.get("properties"); - assertThat(valueProperties.keySet(), containsInAnyOrder("keyword")); - assertLeafs(valueProperties, "keyword"); - } - - { - Map mappings = metadata.getProject() - .findMappings( - new String[] { "index1", "index2", "index3" }, - index -> field -> (index.equals("index2")), - Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP - ); - - assertIndexMappingsNoFields(mappings, "index1"); - assertIndexMappingsNoFields(mappings, "index3"); - assertIndexMappingsNotFiltered(mappings, "index2"); - } - } - - public void testOldestIndexComputation() { - Metadata metadata = buildIndicesWithVersions( - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersion.current(), - IndexVersion.fromId(IndexVersion.current().id() + 1) - ).build(); - - assertEquals(IndexVersions.MINIMUM_COMPATIBLE, metadata.getProject().oldestIndexVersion()); - - Metadata.Builder b = Metadata.builder(); - assertEquals(IndexVersion.current(), b.build().getProject().oldestIndexVersion()); - - Throwable ex = expectThrows( - IllegalArgumentException.class, - () -> buildIndicesWithVersions( - IndexVersions.MINIMUM_COMPATIBLE, - IndexVersions.ZERO, - IndexVersion.fromId(IndexVersion.current().id() + 1) - ).build() - ); - - assertEquals("[index.version.created] is not present in the index settings for index with UUID [null]", ex.getMessage()); - } - - private Metadata.Builder buildIndicesWithVersions(IndexVersion... indexVersions) { - int lastIndexNum = randomIntBetween(9, 50); - Metadata.Builder b = Metadata.builder(); - for (IndexVersion indexVersion : indexVersions) { - IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum)) - .settings(settings(indexVersion)) - .numberOfShards(1) - .numberOfReplicas(1) - .build(); - b.put(im, false); - lastIndexNum = randomIntBetween(lastIndexNum + 1, lastIndexNum + 50); - } - - return b; - } - - private static IndexMetadata.Builder buildIndexMetadata(String name, String alias, Boolean writeIndex) { - return IndexMetadata.builder(name) - .settings(settings(IndexVersion.current())) - .creationDate(randomNonNegativeLong()) - .putAlias(AliasMetadata.builder(alias).writeIndex(writeIndex)) - .numberOfShards(1) - .numberOfReplicas(0); - } - - @SuppressWarnings("unchecked") - private static void assertIndexMappingsNoFields(Map mappings, String index) { - MappingMetadata docMapping = mappings.get(index); - assertNotNull(docMapping); - Map sourceAsMap = docMapping.getSourceAsMap(); - assertThat(sourceAsMap.keySet(), containsInAnyOrder("_routing", "_source", "properties")); - Map typeProperties = (Map) sourceAsMap.get("properties"); - assertThat(typeProperties, anEmptyMap()); - } - - @SuppressWarnings("unchecked") - private static void assertIndexMappingsNotFiltered(Map mappings, String index) { - MappingMetadata docMapping = mappings.get(index); - assertNotNull(docMapping); - - Map sourceAsMap = docMapping.getSourceAsMap(); - assertThat(sourceAsMap.keySet(), containsInAnyOrder("_routing", "_source", "properties")); - - Map typeProperties = (Map) sourceAsMap.get("properties"); - assertThat(typeProperties.keySet(), containsInAnyOrder("name", "address", "birth", "age", "ip", "suggest", "properties")); - - Map name = (Map) typeProperties.get("name"); - assertThat(name.keySet(), containsInAnyOrder("properties")); - Map nameProperties = (Map) name.get("properties"); - assertThat(nameProperties.keySet(), containsInAnyOrder("first", "last")); - assertLeafs(nameProperties, "first", "last"); - - Map address = (Map) typeProperties.get("address"); - assertThat(address.keySet(), containsInAnyOrder("type", "properties")); - Map addressProperties = (Map) address.get("properties"); - assertThat(addressProperties.keySet(), containsInAnyOrder("street", "location", "area")); - assertLeafs(addressProperties, "street", "location", "area"); - - Map properties = (Map) typeProperties.get("properties"); - assertThat(properties.keySet(), containsInAnyOrder("type", "properties")); - Map propertiesProperties = (Map) properties.get("properties"); - assertThat(propertiesProperties.keySet(), containsInAnyOrder("key", "value")); - assertMultiField(propertiesProperties, "key", "keyword"); - assertMultiField(propertiesProperties, "value", "keyword"); - } - - @SuppressWarnings("unchecked") - public static void assertLeafs(Map properties, String... fields) { - assertThat(properties.keySet(), hasItems(fields)); - for (String field : fields) { - Map fieldProp = (Map) properties.get(field); - assertThat(fieldProp, not(hasKey("properties"))); - assertThat(fieldProp, not(hasKey("fields"))); - } - } - - public static void assertMultiField(Map properties, String field, String... subFields) { - assertThat(properties, hasKey(field)); - @SuppressWarnings("unchecked") - Map fieldProp = (Map) properties.get(field); - assertThat(fieldProp, hasKey("fields")); - @SuppressWarnings("unchecked") - Map subFieldsDef = (Map) fieldProp.get("fields"); - assertLeafs(subFieldsDef, subFields); - } - - private static final String FIND_MAPPINGS_TEST_ITEM = """ - { - "_doc": { - "_routing": { - "required":true - }, "_source": { - "enabled":false - }, "properties": { - "name": { - "properties": { - "first": { - "type": "keyword" - }, - "last": { - "type": "keyword" - } - } - }, - "birth": { - "type": "date" - }, - "age": { - "type": "integer" - }, - "ip": { - "type": "ip" - }, - "suggest" : { - "type": "completion" - }, - "address": { - "type": "object", - "properties": { - "street": { - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "area": { - "type": "geo_shape", \s - "tree": "quadtree", - "precision": "1m" - } - } - }, - "properties": { - "type": "nested", - "properties": { - "key" : { - "type": "text", - "fields": { - "keyword" : { - "type" : "keyword" - } - } - }, - "value" : { - "type": "text", - "fields": { - "keyword" : { - "type" : "keyword" - } - } - } - } - } - } - } - } - }"""; - - public void testTransientSettingsOverridePersistentSettings() { - final Setting setting = Setting.simpleString("key"); - final Metadata metadata = Metadata.builder() - .persistentSettings(Settings.builder().put(setting.getKey(), "persistent-value").build()) - .transientSettings(Settings.builder().put(setting.getKey(), "transient-value").build()) - .build(); - assertThat(setting.get(metadata.settings()), equalTo("transient-value")); - } - - public void testBuilderRejectsNullCustom() { - final Metadata.Builder builder = Metadata.builder(); - final String key = randomAlphaOfLength(10); - assertThat( - expectThrows(NullPointerException.class, () -> builder.putCustom(key, (Metadata.ClusterCustom) null)).getMessage(), - containsString(key) - ); - assertThat(expectThrows(NullPointerException.class, () -> builder.putProjectCustom(key, null)).getMessage(), containsString(key)); - } - - public void testBuilderRejectsNullInCustoms() { - final Metadata.Builder builder = Metadata.builder(); - final String key = randomAlphaOfLength(10); - { - final Map map = new HashMap<>(); - map.put(key, null); - assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key)); - } - { - final Map map = new HashMap<>(); - map.put(key, null); - assertThat(expectThrows(NullPointerException.class, () -> builder.projectCustoms(map)).getMessage(), containsString(key)); - } - } - - public void testCopyAndUpdate() throws IOException { - var metadata = Metadata.builder().clusterUUID(UUIDs.base64UUID()).build(); - var newClusterUuid = UUIDs.base64UUID(); - - var copy = metadata.copyAndUpdate(builder -> builder.clusterUUID(newClusterUuid)); - - assertThat(copy, not(sameInstance(metadata))); - assertThat(copy.clusterUUID(), equalTo(newClusterUuid)); - } - - public void testBuilderRemoveClusterCustomIf() { - var custom1 = new TestClusterCustomMetadata(); - var custom2 = new TestClusterCustomMetadata(); - var builder = Metadata.builder(); - builder.putCustom("custom1", custom1); - builder.putCustom("custom2", custom2); - - builder.removeCustomIf((key, value) -> Objects.equals(key, "custom1")); - - var metadata = builder.build(); - assertThat(metadata.custom("custom1"), nullValue()); - assertThat(metadata.custom("custom2"), sameInstance(custom2)); - } - - public void testBuilderRejectsDataStreamThatConflictsWithIndex() { - final String dataStreamName = "my-data-stream"; - IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); - Metadata.Builder b = Metadata.builder() - .put(idx, false) - .put( - IndexMetadata.builder(dataStreamName) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - .build(), - false - ) - .put(newInstance(dataStreamName, List.of(idx.getIndex()))); - - IllegalStateException e = expectThrows(IllegalStateException.class, b::build); - assertThat( - e.getMessage(), - containsString( - "index, alias, and data stream names need to be unique, but the following duplicates were found [data " - + "stream [" - + dataStreamName - + "] conflicts with index]" - ) - ); - } - - public void testBuilderRejectsDataStreamThatConflictsWithAlias() { - final String dataStreamName = "my-data-stream"; - IndexMetadata idx = createFirstBackingIndex(dataStreamName).putAlias(AliasMetadata.builder(dataStreamName).build()).build(); - Metadata.Builder b = Metadata.builder().put(idx, false).put(newInstance(dataStreamName, List.of(idx.getIndex()))); - - IllegalStateException e = expectThrows(IllegalStateException.class, b::build); - assertThat( - e.getMessage(), - containsString( - "index, alias, and data stream names need to be unique, but the following duplicates were found [" - + dataStreamName - + " (alias of [" - + idx.getIndex().getName() - + "]) conflicts with data stream]" - ) - ); - } - - public void testBuilderRejectsAliasThatRefersToDataStreamBackingIndex() { - final String dataStreamName = "my-data-stream"; - final String conflictingName = DataStream.getDefaultBackingIndexName(dataStreamName, 2); - IndexMetadata idx = createFirstBackingIndex(dataStreamName).putAlias(new AliasMetadata.Builder(conflictingName)).build(); - Metadata.Builder b = Metadata.builder().put(idx, false).put(newInstance(dataStreamName, List.of(idx.getIndex()))); - - AssertionError e = expectThrows(AssertionError.class, b::build); - assertThat(e.getMessage(), containsString("aliases [" + conflictingName + "] cannot refer to backing indices of data streams")); - } - - public void testBuilderForDataStreamWithRandomlyNumberedBackingIndices() { - final String dataStreamName = "my-data-stream"; - final List backingIndices = new ArrayList<>(); - final int numBackingIndices = randomIntBetween(2, 5); - int lastBackingIndexNum = 0; - Metadata.Builder b = Metadata.builder(); - for (int k = 1; k <= numBackingIndices; k++) { - lastBackingIndexNum = randomIntBetween(lastBackingIndexNum + 1, lastBackingIndexNum + 50); - IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, lastBackingIndexNum)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - .build(); - b.put(im, false); - backingIndices.add(im.getIndex()); - } - - b.put(newInstance(dataStreamName, backingIndices, lastBackingIndexNum, null)); - Metadata metadata = b.build(); - assertThat(metadata.getProject().dataStreams().keySet(), containsInAnyOrder(dataStreamName)); - assertThat(metadata.getProject().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); - } - - public void testBuildIndicesLookupForDataStreams() { - Metadata.Builder b = Metadata.builder(); - int numDataStreams = randomIntBetween(2, 8); - for (int i = 0; i < numDataStreams; i++) { - String name = "data-stream-" + i; - addDataStream(name, b); - } - - Metadata metadata = b.build(); - assertThat(metadata.getProject().dataStreams().size(), equalTo(numDataStreams)); - for (int i = 0; i < numDataStreams; i++) { - String name = "data-stream-" + i; - IndexAbstraction value = metadata.getProject().getIndicesLookup().get(name); - assertThat(value, notNullValue()); - DataStream ds = metadata.getProject().dataStreams().get(name); - assertThat(ds, notNullValue()); - - assertThat(value.isHidden(), is(false)); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - assertThat(value.getIndices(), hasSize(ds.getIndices().size())); - assertThat(value.getWriteIndex().getName(), DataStreamTestHelper.backingIndexEqualTo(name, (int) ds.getGeneration())); - } - } - - public void testBuildIndicesLookupForDataStreamAliases() { - Metadata.Builder b = Metadata.builder(); - - addDataStream("d1", b); - addDataStream("d2", b); - addDataStream("d3", b); - addDataStream("d4", b); - - b.put("a1", "d1", null, null); - b.put("a1", "d2", null, null); - b.put("a2", "d3", null, null); - b.put("a3", "d1", null, null); - - Metadata metadata = b.build(); - assertThat(metadata.getProject().dataStreams(), aMapWithSize(4)); - IndexAbstraction value = metadata.getProject().getIndicesLookup().get("d1"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - - value = metadata.getProject().getIndicesLookup().get("d2"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - - value = metadata.getProject().getIndicesLookup().get("d3"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - - value = metadata.getProject().getIndicesLookup().get("d4"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); - - value = metadata.getProject().getIndicesLookup().get("a1"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - - value = metadata.getProject().getIndicesLookup().get("a2"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - - value = metadata.getProject().getIndicesLookup().get("a3"); - assertThat(value, notNullValue()); - assertThat(value.getType(), equalTo(IndexAbstraction.Type.ALIAS)); - } - - public void testDataStreamAliasValidation() { - Metadata.Builder b = Metadata.builder(); - addDataStream("my-alias", b); - b.put("my-alias", "my-alias", null, null); - var e = expectThrows(IllegalStateException.class, b::build); - assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); - - b = Metadata.builder(); - addDataStream("d1", b); - addDataStream("my-alias", b); - b.put("my-alias", "d1", null, null); - e = expectThrows(IllegalStateException.class, b::build); - assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); - - b = Metadata.builder(); - b.put( - IndexMetadata.builder("index1") - .settings(indexSettings(IndexVersion.current(), 1, 0)) - .putAlias(new AliasMetadata.Builder("my-alias")) - ); - - addDataStream("d1", b); - b.put("my-alias", "d1", null, null); - e = expectThrows(IllegalStateException.class, b::build); - assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)")); - } - - public void testDataStreamAliasValidationRestoreScenario() { - Metadata.Builder b = Metadata.builder(); - b.dataStreams( - Map.of("my-alias", createDataStream("my-alias")), - Map.of("my-alias", new DataStreamAlias("my-alias", List.of("my-alias"), null, null)) - ); - var e = expectThrows(IllegalStateException.class, b::build); - assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); - - b = Metadata.builder(); - b.dataStreams( - Map.of("d1", createDataStream("d1"), "my-alias", createDataStream("my-alias")), - Map.of("my-alias", new DataStreamAlias("my-alias", List.of("d1"), null, null)) - ); - e = expectThrows(IllegalStateException.class, b::build); - assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); - - b = Metadata.builder(); - b.put( - IndexMetadata.builder("index1") - .settings(indexSettings(IndexVersion.current(), 1, 0)) - .putAlias(new AliasMetadata.Builder("my-alias")) - ); - b.dataStreams(Map.of("d1", createDataStream("d1")), Map.of("my-alias", new DataStreamAlias("my-alias", List.of("d1"), null, null))); - e = expectThrows(IllegalStateException.class, b::build); - assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)")); - } - - private void addDataStream(String name, Metadata.Builder b) { - int numBackingIndices = randomIntBetween(1, 4); - List indices = new ArrayList<>(numBackingIndices); - for (int j = 1; j <= numBackingIndices; j++) { - IndexMetadata idx = createBackingIndex(name, j).build(); - indices.add(idx.getIndex()); - b.put(idx, true); - } - b.put(newInstance(name, indices)); - } - - private DataStream createDataStream(String name) { - int numBackingIndices = randomIntBetween(1, 4); - List indices = new ArrayList<>(numBackingIndices); - for (int j = 1; j <= numBackingIndices; j++) { - IndexMetadata idx = createBackingIndex(name, j).build(); - indices.add(idx.getIndex()); - } - return newInstance(name, indices); - } - - public void testIndicesLookupRecordsDataStreamForBackingIndices() { - final int numIndices = randomIntBetween(2, 5); - final int numBackingIndices = randomIntBetween(2, 5); - final String dataStreamName = "my-data-stream"; - CreateIndexResult result = createIndices(numIndices, numBackingIndices, dataStreamName); - - SortedMap indicesLookup = result.metadata.getProject().getIndicesLookup(); - assertThat(indicesLookup, aMapWithSize(result.indices.size() + result.backingIndices.size() + 1)); - for (Index index : result.indices) { - assertThat(indicesLookup, hasKey(index.getName())); - assertNull(indicesLookup.get(index.getName()).getParentDataStream()); - } - for (Index index : result.backingIndices) { - assertThat(indicesLookup, hasKey(index.getName())); - assertNotNull(indicesLookup.get(index.getName()).getParentDataStream()); - assertThat(indicesLookup.get(index.getName()).getParentDataStream().getName(), equalTo(dataStreamName)); - } - } - - public void testSerialization() throws IOException { - final Metadata orig = randomMetadata(); - final BytesStreamOutput out = new BytesStreamOutput(); - orig.writeTo(out); - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); - final Metadata fromStreamMeta = Metadata.readFrom( - new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) - ); - assertTrue(Metadata.isGlobalStateEquals(orig, fromStreamMeta)); - } - - public void testMultiProjectSerialization() throws IOException { - // TODO: this whole suite needs to be updated for multiple projects - ProjectMetadata project1 = randomProject(ProjectId.fromId("1"), 1); - ProjectMetadata project2 = randomProject(ProjectId.fromId("2"), randomIntBetween(2, 10)); - Metadata metadata = randomMetadata(List.of(project1, project2)); - BytesStreamOutput out = new BytesStreamOutput(); - metadata.writeTo(out); - - // check it deserializes ok - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); - Metadata fromStreamMeta = Metadata.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry)); - - // check it matches the original object - assertThat(fromStreamMeta.projects(), aMapWithSize(2)); - for (var original : List.of(project1, project2)) { - assertThat(fromStreamMeta.projects(), hasKey(original.id())); - final ProjectMetadata fromStreamProject = fromStreamMeta.getProject(original.id()); - assertThat("For project " + original.id(), fromStreamProject.indices().size(), equalTo(original.indices().size())); - assertThat("For project " + original.id(), fromStreamProject.dataStreams().size(), equalTo(original.dataStreams().size())); - assertThat("For project " + original.id(), fromStreamProject.templates().size(), equalTo(original.templates().size())); - assertThat("For project " + original.id(), fromStreamProject.templatesV2().size(), equalTo(original.templatesV2().size())); - original.indices().forEach((name, value) -> { - assertThat(fromStreamProject.indices(), hasKey(name)); - assertThat(fromStreamProject.index(name), equalTo(value)); - }); - original.dataStreams().forEach((name, value) -> { - assertThat(fromStreamProject.dataStreams(), hasKey(name)); - assertThat(fromStreamProject.dataStreams().get(name), equalTo(value)); - }); - } - } - - public void testGetNonExistingProjectThrows() { - final List projects = IntStream.range(0, between(1, 3)) - .mapToObj(i -> randomProject(ProjectId.fromId("p_" + i), between(0, 5))) - .toList(); - final Metadata metadata = randomMetadata(projects); - expectThrows(IllegalArgumentException.class, () -> metadata.getProject(randomProjectIdOrDefault())); - } - - public void testValidateDataStreamsNoConflicts() { - Metadata metadata = createIndices(5, 10, "foo-datastream").metadata; - // don't expect any exception when validating a system without indices that would conflict with future backing indices - assertDataStreams( - metadata.getProject().indices(), - (DataStreamMetadata) metadata.getProject().customs().get(DataStreamMetadata.TYPE) - ); - } - - public void testValidateDataStreamsIgnoresIndicesWithoutCounter() { - String dataStreamName = "foo-datastream"; - Metadata metadata = Metadata.builder(createIndices(10, 10, dataStreamName).metadata) - .put( - new IndexMetadata.Builder(dataStreamName + "-index-without-counter").settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - ) - .put( - new IndexMetadata.Builder(dataStreamName + randomAlphaOfLength(10)).settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - - ) - .put( - new IndexMetadata.Builder(randomAlphaOfLength(10)).settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - - ) - .build(); - // don't expect any exception when validating against non-backing indices that don't conform to the backing indices naming - // convention - assertDataStreams( - metadata.getProject().indices(), - (DataStreamMetadata) metadata.getProject().customs().get(DataStreamMetadata.TYPE) - ); - } - - public void testValidateDataStreamsAllowsNamesThatStartsWithPrefix() { - String dataStreamName = "foo-datastream"; - Metadata metadata = Metadata.builder(createIndices(10, 10, dataStreamName).metadata) - .put( - new IndexMetadata.Builder(DataStream.BACKING_INDEX_PREFIX + dataStreamName + "-something-100012").settings( - settings(IndexVersion.current()) - ).numberOfShards(1).numberOfReplicas(1) - ) - .build(); - // don't expect any exception when validating against (potentially backing) indices that can't create conflict because of - // additional text before number - assertDataStreams( - metadata.getProject().indices(), - (DataStreamMetadata) metadata.getProject().customs().get(DataStreamMetadata.TYPE) - ); - } - - public void testValidateDataStreamsForNullDataStreamMetadata() { - Metadata metadata = Metadata.builder() - .put(IndexMetadata.builder("foo-index").settings(settings(IndexVersion.current())).numberOfShards(1).numberOfReplicas(1)) - .build(); - - try { - assertDataStreams(metadata.getProject().indices(), DataStreamMetadata.EMPTY); - } catch (Exception e) { - fail("did not expect exception when validating a system without any data streams but got " + e.getMessage()); - } - } - - public void testDataStreamAliases() { - Metadata.Builder mdBuilder = Metadata.builder(); - - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null), is(true)); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-us", null, null), is(true)); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au", null, null), is(true)); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au", null, null), is(false)); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") - ); - } - - public void testDataStreamReferToNonExistingDataStream() { - Metadata.Builder mdBuilder = Metadata.builder(); - - Exception e = expectThrows(IllegalArgumentException.class, () -> mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null)); - assertThat(e.getMessage(), equalTo("alias [logs-postgres] refers to a non existing data stream [logs-postgres-eu]")); - - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); - mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); - } - - public void testDeleteDataStreamShouldUpdateAlias() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); - mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); - mdBuilder.put("logs-postgres", "logs-postgres-us", null, null); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); - mdBuilder.put("logs-postgres", "logs-postgres-au", null, null); - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") - ); - - mdBuilder = Metadata.builder(metadata); - mdBuilder.removeDataStream("logs-postgres-us"); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-eu", "logs-postgres-au") - ); - - mdBuilder = Metadata.builder(metadata); - mdBuilder.removeDataStream("logs-postgres-au"); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); - - mdBuilder = Metadata.builder(metadata); - mdBuilder.removeDataStream("logs-postgres-eu"); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), nullValue()); - } - - public void testDeleteDataStreamAlias() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); - mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); - mdBuilder.put("logs-postgres", "logs-postgres-us", null, null); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); - mdBuilder.put("logs-postgres", "logs-postgres-au", null, null); - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") - ); - - mdBuilder = Metadata.builder(metadata); - assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-us", true), is(true)); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-eu", "logs-postgres-au") - ); - - mdBuilder = Metadata.builder(metadata); - assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-au", true), is(true)); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); - - mdBuilder = Metadata.builder(metadata); - assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-eu", true), is(true)); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), nullValue()); - } - - public void testDeleteDataStreamAliasMustExists() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); - mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); - mdBuilder.put("logs-postgres", "logs-postgres-us", null, null); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); - mdBuilder.put("logs-postgres", "logs-postgres-au", null, null); - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") - ); - - Metadata.Builder mdBuilder2 = Metadata.builder(metadata); - expectThrows(ResourceNotFoundException.class, () -> mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", true)); - assertThat(mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", false), is(false)); - } - - public void testDataStreamWriteAlias() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); - mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-replicated") - ); - - mdBuilder = Metadata.builder(metadata); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", true, null), is(true)); - - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), - equalTo("logs-postgres-replicated") - ); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-replicated") - ); - } - - public void testDataStreamMultipleWriteAlias() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-foobar")); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-barbaz")); - mdBuilder.put("logs", "logs-foobar", true, null); - mdBuilder.put("logs", "logs-barbaz", true, null); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs").getWriteDataStream(), equalTo("logs-barbaz")); - assertThat( - metadata.getProject().dataStreamAliases().get("logs").getDataStreams(), - containsInAnyOrder("logs-foobar", "logs-barbaz") - ); - } - - public void testDataStreamWriteAliasUnset() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); - mdBuilder.put("logs-postgres", "logs-postgres-replicated", true, null); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), - equalTo("logs-postgres-replicated") - ); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-replicated") - ); - - mdBuilder = Metadata.builder(metadata); - // Side check: null value isn't changing anything: - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(false)); - // Unset write flag - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", false, null), is(true)); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-replicated") - ); - } - - public void testDataStreamWriteAliasChange() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-primary")); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", true, null), is(true)); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(true)); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-primary")); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") - ); - - // change write flag: - mdBuilder = Metadata.builder(metadata); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", false, null), is(true)); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", true, null), is(true)); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), - equalTo("logs-postgres-replicated") - ); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") - ); - } - - public void testDataStreamWriteRemoveAlias() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-primary")); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", true, null), is(true)); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(true)); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-primary")); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") - ); - - mdBuilder = Metadata.builder(metadata); - assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-primary", randomBoolean()), is(true)); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-replicated") - ); - } - - public void testDataStreamWriteRemoveDataStream() { - Metadata.Builder mdBuilder = Metadata.builder(); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-primary")); - mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", true, null), is(true)); - assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(true)); - - Metadata metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-primary")); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") - ); - - mdBuilder = Metadata.builder(metadata); - mdBuilder.removeDataStream("logs-postgres-primary"); - metadata = mdBuilder.build(); - assertThat(metadata.getProject().dataStreams().keySet(), contains("logs-postgres-replicated")); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres"), notNullValue()); - assertThat(metadata.getProject().dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); - assertThat( - metadata.getProject().dataStreamAliases().get("logs-postgres").getDataStreams(), - containsInAnyOrder("logs-postgres-replicated") - ); - } - - public void testReuseIndicesLookup() { - String indexName = "my-index"; - String aliasName = "my-alias"; - String dataStreamName = "logs-mysql-prod"; - String dataStreamAliasName = "logs-mysql"; - Metadata previous = Metadata.builder().build(); - - // Things that should change indices lookup - { - Metadata.Builder builder = Metadata.builder(previous); - IndexMetadata idx = DataStreamTestHelper.createFirstBackingIndex(dataStreamName).build(); - builder.put(idx, true); - DataStream dataStream = newInstance(dataStreamName, List.of(idx.getIndex())); - builder.put(dataStream); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - builder.put(dataStreamAliasName, dataStreamName, false, null); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - builder.put(dataStreamAliasName, dataStreamName, true, null); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - builder.put( - IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current())) - .creationDate(randomNonNegativeLong()) - .numberOfShards(1) - .numberOfReplicas(0) - ); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); - imBuilder.putAlias(AliasMetadata.builder(aliasName).build()); - builder.put(imBuilder); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); - imBuilder.putAlias(AliasMetadata.builder(aliasName).writeIndex(true).build()); - builder.put(imBuilder); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); - Settings.Builder sBuilder = Settings.builder() - .put(builder.get(indexName).getSettings()) - .put(IndexMetadata.INDEX_HIDDEN_SETTING.getKey(), true); - imBuilder.settings(sBuilder.build()); - builder.put(imBuilder); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), not(sameInstance(metadata.getProject().getIndicesLookup()))); - previous = metadata; - } - - // Things that shouldn't change indices lookup - { - Metadata.Builder builder = Metadata.builder(previous); - IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); - imBuilder.numberOfReplicas(2); - builder.put(imBuilder); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), sameInstance(metadata.getProject().getIndicesLookup())); - previous = metadata; - } - { - Metadata.Builder builder = Metadata.builder(previous); - IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); - Settings.Builder sBuilder = Settings.builder() - .put(builder.get(indexName).getSettings()) - .put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), "val"); - imBuilder.settings(sBuilder.build()); - builder.put(imBuilder); - Metadata metadata = builder.build(); - assertThat(previous.getProject().getIndicesLookup(), sameInstance(metadata.getProject().getIndicesLookup())); - previous = metadata; - } - } - - public void testAliasedIndices() { - int numAliases = randomIntBetween(32, 64); - int numIndicesPerAlias = randomIntBetween(8, 16); - - Metadata.Builder builder = Metadata.builder(); - for (int i = 0; i < numAliases; i++) { - String aliasName = "alias-" + i; - for (int j = 0; j < numIndicesPerAlias; j++) { - AliasMetadata.Builder alias = new AliasMetadata.Builder(aliasName); - if (j == 0) { - alias.writeIndex(true); - } - - String indexName = aliasName + "-" + j; - builder.put( - IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current())) - .creationDate(randomNonNegativeLong()) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(alias) - ); - } - } - - Metadata metadata = builder.build(); - for (int i = 0; i < numAliases; i++) { - String aliasName = "alias-" + i; - Set result = metadata.getProject().aliasedIndices(aliasName); - Index[] expected = IntStream.range(0, numIndicesPerAlias) - .mapToObj(j -> aliasName + "-" + j) - .map(name -> new Index(name, ClusterState.UNKNOWN_UUID)) - .toArray(Index[]::new); - assertThat(result, containsInAnyOrder(expected)); - } - - // Add a new alias and index - builder = Metadata.builder(metadata); - String newAliasName = "alias-new"; - { - builder.put( - IndexMetadata.builder(newAliasName + "-1") - .settings(settings(IndexVersion.current())) - .creationDate(randomNonNegativeLong()) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(new AliasMetadata.Builder(newAliasName).writeIndex(true)) - ); - } - metadata = builder.build(); - assertThat(metadata.getProject().aliasedIndices(), hasSize(numAliases + 1)); - assertThat(metadata.getProject().aliasedIndices(newAliasName), contains(new Index(newAliasName + "-1", ClusterState.UNKNOWN_UUID))); - - // Remove the new alias/index - builder = Metadata.builder(metadata); - { - builder.remove(newAliasName + "-1"); - } - metadata = builder.build(); - assertThat(metadata.getProject().aliasedIndices(), hasSize(numAliases)); - assertThat(metadata.getProject().aliasedIndices(newAliasName), empty()); - - // Add a new alias that points to existing indices - builder = Metadata.builder(metadata); - { - IndexMetadata.Builder imBuilder = new IndexMetadata.Builder(metadata.getProject().index("alias-1-0")); - imBuilder.putAlias(new AliasMetadata.Builder(newAliasName)); - builder.put(imBuilder); - - imBuilder = new IndexMetadata.Builder(metadata.getProject().index("alias-2-1")); - imBuilder.putAlias(new AliasMetadata.Builder(newAliasName)); - builder.put(imBuilder); - - imBuilder = new IndexMetadata.Builder(metadata.getProject().index("alias-3-2")); - imBuilder.putAlias(new AliasMetadata.Builder(newAliasName)); - builder.put(imBuilder); - } - metadata = builder.build(); - assertThat(metadata.getProject().aliasedIndices(), hasSize(numAliases + 1)); - assertThat( - metadata.getProject().aliasedIndices(newAliasName), - containsInAnyOrder( - new Index("alias-1-0", ClusterState.UNKNOWN_UUID), - new Index("alias-2-1", ClusterState.UNKNOWN_UUID), - new Index("alias-3-2", ClusterState.UNKNOWN_UUID) - ) - ); - - // Remove the new alias that points to existing indices - builder = Metadata.builder(metadata); - { - IndexMetadata.Builder imBuilder = new IndexMetadata.Builder(metadata.getProject().index("alias-1-0")); - imBuilder.removeAlias(newAliasName); - builder.put(imBuilder); - - imBuilder = new IndexMetadata.Builder(metadata.getProject().index("alias-2-1")); - imBuilder.removeAlias(newAliasName); - builder.put(imBuilder); - - imBuilder = new IndexMetadata.Builder(metadata.getProject().index("alias-3-2")); - imBuilder.removeAlias(newAliasName); - builder.put(imBuilder); - } - metadata = builder.build(); - assertThat(metadata.getProject().aliasedIndices(), hasSize(numAliases)); - assertThat(metadata.getProject().aliasedIndices(newAliasName), empty()); - } - - public static final String SYSTEM_ALIAS_NAME = "system_alias"; - - public void testHiddenAliasValidation() { - final String hiddenAliasName = "hidden_alias"; - - IndexMetadata hidden1 = buildIndexWithAlias("hidden1", hiddenAliasName, true, IndexVersion.current(), false); - IndexMetadata hidden2 = buildIndexWithAlias("hidden2", hiddenAliasName, true, IndexVersion.current(), false); - IndexMetadata hidden3 = buildIndexWithAlias("hidden3", hiddenAliasName, true, IndexVersion.current(), false); - - IndexMetadata nonHidden = buildIndexWithAlias("nonhidden1", hiddenAliasName, false, IndexVersion.current(), false); - IndexMetadata unspecified = buildIndexWithAlias("nonhidden2", hiddenAliasName, null, IndexVersion.current(), false); - - { - // Should be ok: - metadataWithIndices(hidden1, hidden2, hidden3); - } - - { - // Should be ok: - if (randomBoolean()) { - metadataWithIndices(nonHidden, unspecified); - } else { - metadataWithIndices(unspecified, nonHidden); + "pending_generation": 42 + } + }, + "reserved_state":{ } + } } - } - - { - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> metadataWithIndices(hidden1, hidden2, hidden3, nonHidden) - ); - assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] has is_hidden set to true on indices [")); - assertThat( - exception.getMessage(), - allOf( - containsString(hidden1.getIndex().getName()), - containsString(hidden2.getIndex().getName()), - containsString(hidden3.getIndex().getName()) - ) - ); - assertThat( - exception.getMessage(), - containsString( - "but does not have is_hidden set to true on indices [" - + nonHidden.getIndex().getName() - + "]; alias must have the same is_hidden setting on all indices" - ) - ); - } - - { - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> metadataWithIndices(hidden1, hidden2, hidden3, unspecified) - ); - assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] has is_hidden set to true on indices [")); - assertThat( - exception.getMessage(), - allOf( - containsString(hidden1.getIndex().getName()), - containsString(hidden2.getIndex().getName()), - containsString(hidden3.getIndex().getName()) - ) - ); - assertThat( - exception.getMessage(), - containsString( - "but does not have is_hidden set to true on indices [" - + unspecified.getIndex().getName() - + "]; alias must have the same is_hidden setting on all indices" - ) - ); - } - - { - final IndexMetadata hiddenIndex = randomFrom(hidden1, hidden2, hidden3); - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { - if (randomBoolean()) { - metadataWithIndices(nonHidden, unspecified, hiddenIndex); - } else { - metadataWithIndices(unspecified, nonHidden, hiddenIndex); - } - }); - assertThat( - exception.getMessage(), - containsString( - "alias [" - + hiddenAliasName - + "] has is_hidden set to true on " - + "indices [" - + hiddenIndex.getIndex().getName() - + "] but does not have is_hidden set to true on indices [" - ) - ); - assertThat( - exception.getMessage(), - allOf(containsString(unspecified.getIndex().getName()), containsString(nonHidden.getIndex().getName())) - ); - assertThat(exception.getMessage(), containsString("but does not have is_hidden set to true on indices [")); - } - } + """, IndexVersion.current(), IndexVersion.current()); - public void testSystemAliasValidationMixedVersionSystemAndRegularFails() { - final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_0_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) - ); - final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); - final IndexMetadata regularIndex = buildIndexWithAlias("regular1", SYSTEM_ALIAS_NAME, false, IndexVersion.current(), false); + final Metadata metadata = fromJsonXContentStringWithPersistentTasks(json); + assertThat(metadata, notNullValue()); + assertThat(metadata.clusterUUID(), is("aba1aa1ababbbaabaabaab")); - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> metadataWithIndices(currentVersionSystem, oldVersionSystem, regularIndex) - ); + assertThat(metadata.projects().keySet(), containsInAnyOrder(ProjectId.fromId("default"), ProjectId.fromId("another_project"))); + assertThat(metadata.customs(), not(hasKey("repositories"))); + final var repositoriesMetadata = RepositoriesMetadata.get(metadata.getProject(ProjectId.DEFAULT)); assertThat( - exception.getMessage(), - containsString( - "alias [" - + SYSTEM_ALIAS_NAME - + "] refers to both system indices [" - + currentVersionSystem.getIndex().getName() - + "] and non-system indices: [" - + regularIndex.getIndex().getName() - + "], but aliases must refer to either system or non-system indices, not both" + repositoriesMetadata.repositories(), + equalTo( + List.of( + new RepositoryMetadata("my-repo", "_my-repo-uuid_", "fs", Settings.builder().put("location", "backup").build(), 42, 42) + ) ) ); + assertThat(metadata.getProject(ProjectId.fromId("another_project")).customs(), not(hasKey("repositories"))); } - public void testSystemAliasValidationNewSystemAndRegularFails() { - final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata regularIndex = buildIndexWithAlias("regular1", SYSTEM_ALIAS_NAME, false, IndexVersion.current(), false); - - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> metadataWithIndices(currentVersionSystem, regularIndex) - ); - assertThat( - exception.getMessage(), - containsString( - "alias [" - + SYSTEM_ALIAS_NAME - + "] refers to both system indices [" - + currentVersionSystem.getIndex().getName() - + "] and non-system indices: [" - + regularIndex.getIndex().getName() - + "], but aliases must refer to either system or non-system indices, not both" - ) - ); - } + private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws IOException { + List registry = new ArrayList<>(); + registry.addAll(ClusterModule.getNamedXWriteables()); + registry.addAll(IndicesModule.getNamedXContents()); + registry.addAll(HealthNodeTaskExecutor.getNamedXContentParsers()); - public void testSystemAliasOldSystemAndNewRegular() { - final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_0_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + final var clusterService = mock(ClusterService.class); + when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class)); + final var healthNodeTaskExecutor = HealthNodeTaskExecutor.create( + clusterService, + mock(PersistentTasksService.class), + Settings.EMPTY, + ClusterSettings.createBuiltInClusterSettings() ); - final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); - final IndexMetadata regularIndex = buildIndexWithAlias("regular1", SYSTEM_ALIAS_NAME, false, IndexVersion.current(), false); + new PersistentTasksExecutorRegistry(List.of(healthNodeTaskExecutor)); - // Should be ok: - metadataWithIndices(oldVersionSystem, regularIndex); + XContentParserConfiguration config = XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(registry)); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(config, json)) { + return Metadata.fromXContent(parser); + } } - public void testSystemIndexValidationAllRegular() { - final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_0_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + public void testGlobalStateEqualsCoordinationMetadata() { + CoordinationMetadata coordinationMetadata1 = new CoordinationMetadata( + randomNonNegativeLong(), + randomVotingConfig(), + randomVotingConfig(), + randomVotingConfigExclusions() ); - final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata currentVersionSystem2 = buildIndexWithAlias(".system2", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); - - // Should be ok - metadataWithIndices(currentVersionSystem, currentVersionSystem2, oldVersionSystem); - } - - public void testSystemAliasValidationAllSystemSomeOld() { - final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_0_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + Metadata metadata1 = Metadata.builder().coordinationMetadata(coordinationMetadata1).build(); + CoordinationMetadata coordinationMetadata2 = new CoordinationMetadata( + randomNonNegativeLong(), + randomVotingConfig(), + randomVotingConfig(), + randomVotingConfigExclusions() ); - final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata currentVersionSystem2 = buildIndexWithAlias(".system2", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); + Metadata metadata2 = Metadata.builder().coordinationMetadata(coordinationMetadata2).build(); - // Should be ok: - metadataWithIndices(currentVersionSystem, currentVersionSystem2, oldVersionSystem); + assertTrue(Metadata.isGlobalStateEquals(metadata1, metadata1)); + assertFalse(Metadata.isGlobalStateEquals(metadata1, metadata2)); } - public void testSystemAliasValidationAll8x() { - final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - final IndexMetadata currentVersionSystem2 = buildIndexWithAlias(".system2", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); - - // Should be ok - metadataWithIndices(currentVersionSystem, currentVersionSystem2); + public void testSerializationWithIndexGraveyard() throws IOException { + final var projectId = randomProjectIdOrDefault(); + final IndexGraveyard graveyard = IndexGraveyardTests.createRandom(); + final Metadata originalMeta = Metadata.builder().put(ProjectMetadata.builder(projectId).indexGraveyard(graveyard)).build(); + final BytesStreamOutput out = new BytesStreamOutput(); + originalMeta.writeTo(out); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + final Metadata fromStreamMeta = Metadata.readFrom( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) + ); + assertThat(fromStreamMeta.getProject(projectId).indexGraveyard(), equalTo(originalMeta.getProject(projectId).indexGraveyard())); } - private void metadataWithIndices(IndexMetadata... indices) { - Metadata.Builder builder = Metadata.builder(); - for (var cursor : indices) { - builder.put(cursor, false); - } - builder.build(); + private static IndexMetadata.Builder buildIndexMetadata(String name, String alias, Boolean writeIndex) { + return IndexMetadata.builder(name) + .settings(settings(IndexVersion.current())) + .creationDate(randomNonNegativeLong()) + .putAlias(AliasMetadata.builder(alias).writeIndex(writeIndex)) + .numberOfShards(1) + .numberOfReplicas(0); } - private IndexMetadata buildIndexWithAlias( - String indexName, - String aliasName, - @Nullable Boolean aliasIsHidden, - IndexVersion indexCreationVersion, - boolean isSystem - ) { - final AliasMetadata.Builder aliasMetadata = new AliasMetadata.Builder(aliasName); - if (aliasIsHidden != null || randomBoolean()) { - aliasMetadata.isHidden(aliasIsHidden); - } - return new IndexMetadata.Builder(indexName).settings(settings(indexCreationVersion)) - .system(isSystem) - .numberOfShards(1) - .numberOfReplicas(0) - .putAlias(aliasMetadata) + public void testTransientSettingsOverridePersistentSettings() { + final Setting setting = Setting.simpleString("key"); + final Metadata metadata = Metadata.builder() + .persistentSettings(Settings.builder().put(setting.getKey(), "persistent-value").build()) + .transientSettings(Settings.builder().put(setting.getKey(), "transient-value").build()) .build(); + assertThat(setting.get(metadata.settings()), equalTo("transient-value")); } - public void testMappingDuplication() { - final Set randomMappingDefinitions; - { - int numEntries = randomIntBetween(4, 8); - randomMappingDefinitions = Sets.newHashSetWithExpectedSize(numEntries); - for (int i = 0; i < numEntries; i++) { - Map mapping = RandomAliasActionsGenerator.randomMap(2); - String mappingAsString = Strings.toString((builder, params) -> builder.mapContents(mapping)); - randomMappingDefinitions.add(mappingAsString); - } - } - - Metadata metadata; - int numIndices = randomIntBetween(16, 32); - { - String[] definitions = randomMappingDefinitions.toArray(String[]::new); - Metadata.Builder mb = new Metadata.Builder(); - for (int i = 0; i < numIndices; i++) { - IndexMetadata.Builder indexBuilder = IndexMetadata.builder("index-" + i) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .putMapping(definitions[i % randomMappingDefinitions.size()]) - .numberOfShards(1) - .numberOfReplicas(0); - if (randomBoolean()) { - mb.put(indexBuilder); - } else { - mb.put(indexBuilder.build(), true); - } - } - metadata = mb.build(); - } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); + public void testBuilderRejectsNullCustom() { + final Metadata.Builder builder = Metadata.builder(); + final String key = randomAlphaOfLength(10); assertThat( - metadata.getProject().indices().values().stream().map(IndexMetadata::mapping).collect(Collectors.toSet()), - hasSize(metadata.getProject().getMappingsByHash().size()) + expectThrows(NullPointerException.class, () -> builder.putCustom(key, (Metadata.ClusterCustom) null)).getMessage(), + containsString(key) ); + } - // Add a new index with a new index with known mapping: - MappingMetadata mapping = metadata.getProject().indices().get("index-" + randomInt(numIndices - 1)).mapping(); - MappingMetadata entry = metadata.getProject().getMappingsByHash().get(mapping.getSha256()); + public void testBuilderRejectsNullInCustoms() { + final Metadata.Builder builder = Metadata.builder(); + final String key = randomAlphaOfLength(10); { - Metadata.Builder mb = new Metadata.Builder(metadata); - mb.put( - IndexMetadata.builder("index-" + numIndices) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .putMapping(mapping) - .numberOfShards(1) - .numberOfReplicas(0) - ); - metadata = mb.build(); + final Map map = new HashMap<>(); + map.put(key, null); + assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key)); } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); - assertThat(metadata.getProject().getMappingsByHash().get(mapping.getSha256()), equalTo(entry)); + } - // Remove index and ensure mapping cache stays the same - { - Metadata.Builder mb = new Metadata.Builder(metadata); - mb.remove("index-" + numIndices); - metadata = mb.build(); - } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); - assertThat(metadata.getProject().getMappingsByHash().get(mapping.getSha256()), equalTo(entry)); + public void testCopyAndUpdate() throws IOException { + var metadata = Metadata.builder().clusterUUID(UUIDs.base64UUID()).build(); + var newClusterUuid = UUIDs.base64UUID(); - // Update a mapping of an index: - IndexMetadata luckyIndex = metadata.getProject().index("index-" + randomInt(numIndices - 1)); - entry = metadata.getProject().getMappingsByHash().get(luckyIndex.mapping().getSha256()); - MappingMetadata updatedMapping = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Map.of("mapping", "updated")); - { - Metadata.Builder mb = new Metadata.Builder(metadata); - mb.put(IndexMetadata.builder(luckyIndex).putMapping(updatedMapping)); - metadata = mb.build(); - } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size() + 1)); - assertThat(metadata.getProject().getMappingsByHash().get(luckyIndex.mapping().getSha256()), equalTo(entry)); - assertThat(metadata.getProject().getMappingsByHash().get(updatedMapping.getSha256()), equalTo(updatedMapping)); + var copy = metadata.copyAndUpdate(builder -> builder.clusterUUID(newClusterUuid)); - // Remove the index with updated mapping - { - Metadata.Builder mb = new Metadata.Builder(metadata); - mb.remove(luckyIndex.getIndex().getName()); - metadata = mb.build(); - } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); - assertThat(metadata.getProject().getMappingsByHash().get(updatedMapping.getSha256()), nullValue()); + assertThat(copy, not(sameInstance(metadata))); + assertThat(copy.clusterUUID(), equalTo(newClusterUuid)); + } - // Add an index with new mapping and then later remove it: - MappingMetadata newMapping = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Map.of("new", "mapping")); - { - Metadata.Builder mb = new Metadata.Builder(metadata); - mb.put( - IndexMetadata.builder("index-" + numIndices) - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) - .putMapping(newMapping) - .numberOfShards(1) - .numberOfReplicas(0) - ); - metadata = mb.build(); - } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size() + 1)); - assertThat(metadata.getProject().getMappingsByHash().get(newMapping.getSha256()), equalTo(newMapping)); + public void testBuilderRemoveClusterCustomIf() { + var custom1 = new TestClusterCustomMetadata(); + var custom2 = new TestClusterCustomMetadata(); + var builder = Metadata.builder(); + builder.putCustom("custom1", custom1); + builder.putCustom("custom2", custom2); - { - Metadata.Builder mb = new Metadata.Builder(metadata); - mb.remove("index-" + numIndices); - metadata = mb.build(); + builder.removeCustomIf((key, value) -> Objects.equals(key, "custom1")); + + var metadata = builder.build(); + assertThat(metadata.custom("custom1"), nullValue()); + assertThat(metadata.custom("custom2"), sameInstance(custom2)); + } + + public void testSerialization() throws IOException { + final Metadata orig = randomMetadata(); + final BytesStreamOutput out = new BytesStreamOutput(); + orig.writeTo(out); + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + final Metadata fromStreamMeta = Metadata.readFrom( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry) + ); + assertTrue(Metadata.isGlobalStateEquals(orig, fromStreamMeta)); + } + + public void testMultiProjectSerialization() throws IOException { + ProjectMetadata project1 = randomProject(randomProjectIdOrDefault(), 1); + ProjectMetadata project2 = randomProject(randomUniqueProjectId(), randomIntBetween(2, 10)); + Metadata metadata = randomMetadata(List.of(project1, project2)); + BytesStreamOutput out = new BytesStreamOutput(); + metadata.writeTo(out); + + // check it deserializes ok + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + Metadata fromStreamMeta = Metadata.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry)); + + // check it matches the original object + assertThat(fromStreamMeta.projects(), aMapWithSize(2)); + for (var original : List.of(project1, project2)) { + assertThat(fromStreamMeta.projects(), hasKey(original.id())); + final ProjectMetadata fromStreamProject = fromStreamMeta.getProject(original.id()); + assertThat("For project " + original.id(), fromStreamProject.indices().size(), equalTo(original.indices().size())); + assertThat("For project " + original.id(), fromStreamProject.dataStreams().size(), equalTo(original.dataStreams().size())); + assertThat("For project " + original.id(), fromStreamProject.templates().size(), equalTo(original.templates().size())); + assertThat("For project " + original.id(), fromStreamProject.templatesV2().size(), equalTo(original.templatesV2().size())); + original.indices().forEach((name, value) -> { + assertThat(fromStreamProject.indices(), hasKey(name)); + assertThat(fromStreamProject.index(name), equalTo(value)); + }); + original.dataStreams().forEach((name, value) -> { + assertThat(fromStreamProject.dataStreams(), hasKey(name)); + assertThat(fromStreamProject.dataStreams().get(name), equalTo(value)); + }); } - assertThat(metadata.getProject().getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); - assertThat(metadata.getProject().getMappingsByHash().get(newMapping.getSha256()), nullValue()); } - public void testWithLifecycleState() { - String indexName = "my-index"; - String indexUUID = randomAlphaOfLength(10); - Metadata metadata1 = Metadata.builder(randomMetadata()) - .put( - IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, indexUUID)) - .creationDate(randomNonNegativeLong()) - .numberOfShards(1) - .numberOfReplicas(0) - ) - .build(); - IndexMetadata index1 = metadata1.getProject().index(indexName); - assertThat(metadata1.getProject().getIndicesLookup(), notNullValue()); - assertThat(index1.getLifecycleExecutionState(), sameInstance(LifecycleExecutionState.EMPTY_STATE)); - - LifecycleExecutionState state = LifecycleExecutionState.builder().setPhase("phase").setAction("action").setStep("step").build(); - Metadata metadata2 = metadata1.withLifecycleState(index1.getIndex(), state); - IndexMetadata index2 = metadata2.getProject().index(indexName); - - // the indices lookups are the same object - assertThat(metadata2.getProject().getIndicesLookup(), sameInstance(metadata1.getProject().getIndicesLookup())); - - // the lifecycle state and version were changed - assertThat(index2.getLifecycleExecutionState().asMap(), is(state.asMap())); - assertThat(index2.getVersion(), is(index1.getVersion() + 1)); - - // but those are the only differences between the two - IndexMetadata.Builder builder = IndexMetadata.builder(index2); - builder.version(builder.version() - 1); - builder.removeCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY); - assertThat(index1, equalTo(builder.build())); - - // withLifecycleState returns the same reference if nothing changed - Metadata metadata3 = metadata2.withLifecycleState(index2.getIndex(), state); - assertThat(metadata3, sameInstance(metadata2)); - - // withLifecycleState rejects a nonsense Index - String randomUUID = randomValueOtherThan(indexUUID, () -> randomAlphaOfLength(10)); - expectThrows(IndexNotFoundException.class, () -> metadata1.withLifecycleState(new Index(indexName, randomUUID), state)); + public void testGetNonExistingProjectThrows() { + final List projects = IntStream.range(0, between(1, 3)) + .mapToObj(i -> randomProject(ProjectId.fromId("p_" + i), between(0, 5))) + .toList(); + final Metadata metadata = randomMetadata(projects); + expectThrows(IllegalArgumentException.class, () -> metadata.getProject(randomProjectIdOrDefault())); } public void testEmptyDiffReturnsSameInstance() throws IOException { @@ -3049,90 +1040,6 @@ public void testEnsureMetadataFieldCheckedForGlobalStateChanges() { assertThat(unclassifiedFields, empty()); } - public void testRetrieveIndexModeFromTemplateTsdb() throws IOException { - // tsdb: - var tsdbTemplate = new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("{}"), null); - // Settings in component template: - { - var componentTemplate = new ComponentTemplate(tsdbTemplate, null, null); - var indexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .componentTemplates(List.of("component_template_1")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.getProject().retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES)); - } - // Settings in composable index template: - { - var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); - var indexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(tsdbTemplate) - .componentTemplates(List.of("component_template_1")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.getProject().retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES)); - } - } - - public void testRetrieveIndexModeFromTemplateLogsdb() throws IOException { - // logsdb: - var logsdbTemplate = new Template(Settings.builder().put("index.mode", "logsdb").build(), new CompressedXContent("{}"), null); - // Settings in component template: - { - var componentTemplate = new ComponentTemplate(logsdbTemplate, null, null); - var indexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .componentTemplates(List.of("component_template_1")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.getProject().retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB)); - } - // Settings in composable index template: - { - var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); - var indexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(logsdbTemplate) - .componentTemplates(List.of("component_template_1")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.getProject().retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB)); - } - } - - public void testRetrieveIndexModeFromTemplateEmpty() throws IOException { - // no index mode: - var emptyTemplate = new Template(Settings.EMPTY, new CompressedXContent("{}"), null); - // Settings in component template: - { - var componentTemplate = new ComponentTemplate(emptyTemplate, null, null); - var indexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .componentTemplates(List.of("component_template_1")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.getProject().retrieveIndexModeFromTemplate(indexTemplate), nullValue()); - } - // Settings in composable index template: - { - var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); - var indexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("test-*")) - .template(emptyTemplate) - .componentTemplates(List.of("component_template_1")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) - .build(); - Metadata m = Metadata.builder().put("component_template_1", componentTemplate).put("index_template_1", indexTemplate).build(); - assertThat(m.getProject().retrieveIndexModeFromTemplate(indexTemplate), nullValue()); - } - } - public void testGetSingleProjectWithCustom() { var type = IngestMetadata.TYPE; { @@ -3312,56 +1219,11 @@ public static ProjectMetadata randomProject(ProjectId id, int numDataStreams) { return project.build(); } - private static CreateIndexResult createIndices(int numIndices, int numBackingIndices, String dataStreamName) { - // create some indices that do not back a data stream - final List indices = new ArrayList<>(); - int lastIndexNum = randomIntBetween(9, 50); - Metadata.Builder b = Metadata.builder(); - for (int k = 1; k <= numIndices; k++) { - IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - .build(); - b.put(im, false); - indices.add(im.getIndex()); - lastIndexNum = randomIntBetween(lastIndexNum + 1, lastIndexNum + 50); - } - - // create some backing indices for a data stream - final List backingIndices = new ArrayList<>(); - int lastBackingIndexNum = 0; - for (int k = 1; k <= numBackingIndices; k++) { - lastBackingIndexNum = randomIntBetween(lastBackingIndexNum + 1, lastBackingIndexNum + 50); - IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, lastBackingIndexNum)) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(1) - .build(); - b.put(im, false); - backingIndices.add(im.getIndex()); - } - b.put(newInstance(dataStreamName, backingIndices, lastBackingIndexNum, null)); - return new CreateIndexResult(indices, backingIndices, b.build()); - } - private static ToXContent.Params formatParams() { return new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY)); } - private static class CreateIndexResult { - final List indices; - final List backingIndices; - final Metadata metadata; - - CreateIndexResult(List indices, List backingIndices, Metadata metadata) { - this.indices = indices; - this.backingIndices = backingIndices; - this.metadata = metadata; - } - } - - private abstract static class AbstractCustomMetadata> implements Metadata.MetadataCustom { + private static class TestClusterCustomMetadata implements Metadata.ClusterCustom { @Override public Iterator toXContentChunked(ToXContent.Params params) { @@ -3369,7 +1231,7 @@ public Iterator toXContentChunked(ToXContent.Params params } @Override - public Diff diff(C previousState) { + public Diff diff(Metadata.ClusterCustom previousState) { return null; } @@ -3393,12 +1255,4 @@ public void writeTo(StreamOutput out) throws IOException { } } - - private static class TestClusterCustomMetadata extends AbstractCustomMetadata - implements - Metadata.ClusterCustom {} - - private static class TestProjectCustomMetadata extends AbstractCustomMetadata - implements - Metadata.ProjectCustom {} } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index c39b5caeebce1..bd63b6e70371d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -9,39 +9,2226 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Predicates; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.alias.RandomAliasActionsGenerator; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.plugins.FieldPredicate; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createBackingIndex; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.cluster.metadata.MetadataTests.checkChunkSize; import static org.elasticsearch.cluster.metadata.MetadataTests.count; +import static org.elasticsearch.cluster.metadata.ProjectMetadata.Builder.assertDataStreams; +import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.Matchers.startsWith; public class ProjectMetadataTests extends ESTestCase { - private static final Setting PROJECT_SETTING = Setting.intSetting( - "project.setting.value", - 0, - Setting.Property.Dynamic, - Setting.Property.NodeScope, - Setting.Property.ProjectScope - ); + public void testFindAliases() { + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put( + IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias1").build()) + .putAlias(AliasMetadata.builder("alias2").build()) + ) + .put( + IndexMetadata.builder("index2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias2").build()) + .putAlias(AliasMetadata.builder("alias3").build()) + ) + .build(); + + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT); + Map> aliases = project.findAliases(request.aliases(), Strings.EMPTY_ARRAY); + assertThat(aliases, anEmptyMap()); + } + { + final GetAliasesRequest request; + if (randomBoolean()) { + request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT); + } else { + request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, randomFrom("alias1", "alias2")); + // replacing with empty aliases behaves as if aliases were unspecified at request building + request.replaceAliases(Strings.EMPTY_ARRAY); + } + Map> aliases = project.findAliases(request.aliases(), new String[] { "index" }); + assertThat(aliases, aMapWithSize(1)); + List aliasMetadataList = aliases.get("index"); + assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2"))); + } + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias*"); + Map> aliases = project.findAliases(request.aliases(), new String[] { "index", "index2" }); + assertThat(aliases, aMapWithSize(2)); + List indexAliasMetadataList = aliases.get("index"); + assertThat(indexAliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2"))); + List index2AliasMetadataList = aliases.get("index2"); + assertThat(index2AliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias2", "alias3"))); + } + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT, "alias1"); + Map> aliases = project.findAliases(request.aliases(), new String[] { "index" }); + assertThat(aliases, aMapWithSize(1)); + List aliasMetadataList = aliases.get("index"); + assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1"))); + } + { + Map> aliases = project.findAllAliases(new String[] { "index" }); + assertThat(aliases, aMapWithSize(1)); + List aliasMetadataList = aliases.get("index"); + assertThat(aliasMetadataList, transformedItemsMatch(AliasMetadata::alias, contains("alias1", "alias2"))); + } + { + Map> aliases = project.findAllAliases(Strings.EMPTY_ARRAY); + assertThat(aliases, anEmptyMap()); + } + } + + public void testFindDataStreamAliases() { + ProjectMetadata.Builder builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + + addDataStream("d1", builder); + addDataStream("d2", builder); + addDataStream("d3", builder); + addDataStream("d4", builder); + + builder.put("alias1", "d1", null, null); + builder.put("alias2", "d2", null, null); + builder.put("alias2-part2", "d2", null, null); + + ProjectMetadata project = builder.build(); + + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT); + Map> aliases = project.findDataStreamAliases(request.aliases(), Strings.EMPTY_ARRAY); + assertThat(aliases, anEmptyMap()); + } + + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("alias1"); + Map> aliases = project.findDataStreamAliases(request.aliases(), new String[] { "index" }); + assertThat(aliases, anEmptyMap()); + } + + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("alias1"); + Map> aliases = project.findDataStreamAliases( + request.aliases(), + new String[] { "index", "d1", "d2" } + ); + assertEquals(1, aliases.size()); + List found = aliases.get("d1"); + assertThat(found, transformedItemsMatch(DataStreamAlias::getAlias, contains("alias1"))); + } + + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("ali*"); + Map> aliases = project.findDataStreamAliases(request.aliases(), new String[] { "index", "d2" }); + assertEquals(1, aliases.size()); + List found = aliases.get("d2"); + assertThat(found, transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); + } + + // test exclusion + { + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("*"); + Map> aliases = project.findDataStreamAliases( + request.aliases(), + new String[] { "index", "d1", "d2", "d3", "d4" } + ); + assertThat(aliases.get("d2"), transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); + assertThat(aliases.get("d1"), transformedItemsMatch(DataStreamAlias::getAlias, contains("alias1"))); + + request.aliases("*", "-alias1"); + aliases = project.findDataStreamAliases(request.aliases(), new String[] { "index", "d1", "d2", "d3", "d4" }); + assertThat(aliases.get("d2"), transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); + assertNull(aliases.get("d1")); + } + } + + public void testDataStreamAliasesByDataStream() { + ProjectMetadata.Builder builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + + addDataStream("d1", builder); + addDataStream("d2", builder); + addDataStream("d3", builder); + addDataStream("d4", builder); + + builder.put("alias1", "d1", null, null); + builder.put("alias2", "d2", null, null); + builder.put("alias2-part2", "d2", null, null); + + ProjectMetadata project = builder.build(); + + var aliases = project.dataStreamAliasesByDataStream(); + + assertTrue(aliases.containsKey("d1")); + assertTrue(aliases.containsKey("d2")); + assertFalse(aliases.containsKey("d3")); + assertFalse(aliases.containsKey("d4")); + + assertEquals(1, aliases.get("d1").size()); + assertEquals(2, aliases.get("d2").size()); + + assertThat(aliases.get("d2"), transformedItemsMatch(DataStreamAlias::getAlias, containsInAnyOrder("alias2", "alias2-part2"))); + } + + public void testFindAliasWithExclusion() { + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put( + IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias1").build()) + .putAlias(AliasMetadata.builder("alias2").build()) + ) + .put( + IndexMetadata.builder("index2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias1").build()) + .putAlias(AliasMetadata.builder("alias3").build()) + ) + .build(); + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("*", "-alias1"); + Map> aliases = project.findAliases(request.aliases(), new String[] { "index", "index2" }); + assertThat(aliases.get("index"), transformedItemsMatch(AliasMetadata::alias, contains("alias2"))); + assertThat(aliases.get("index2"), transformedItemsMatch(AliasMetadata::alias, contains("alias3"))); + } + + public void testFindDataStreams() { + final int numIndices = randomIntBetween(2, 5); + final int numBackingIndices = randomIntBetween(2, 5); + final String dataStreamName = "my-data-stream"; + CreateIndexResult result = createIndices(numIndices, numBackingIndices, dataStreamName); + + List allIndices = new ArrayList<>(result.indices); + allIndices.addAll(result.backingIndices); + String[] concreteIndices = allIndices.stream().map(Index::getName).toArray(String[]::new); + Map dataStreams = result.project.findDataStreams(concreteIndices); + assertThat(dataStreams, aMapWithSize(numBackingIndices)); + for (Index backingIndex : result.backingIndices) { + assertThat(dataStreams, hasKey(backingIndex.getName())); + assertThat(dataStreams.get(backingIndex.getName()).getName(), equalTo(dataStreamName)); + } + } + + public void testFindAliasWithExclusionAndOverride() { + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put( + IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("aa").build()) + .putAlias(AliasMetadata.builder("ab").build()) + .putAlias(AliasMetadata.builder("bb").build()) + ) + .build(); + GetAliasesRequest request = new GetAliasesRequest(TEST_REQUEST_TIMEOUT).aliases("a*", "-*b", "b*"); + List aliases = project.findAliases(request.aliases(), new String[] { "index" }).get("index"); + assertThat(aliases, transformedItemsMatch(AliasMetadata::alias, contains("aa", "bb"))); + } + + public void testAliasCollidingWithAnExistingIndex() { + int indexCount = randomIntBetween(10, 100); + Set indices = Sets.newHashSetWithExpectedSize(indexCount); + for (int i = 0; i < indexCount; i++) { + indices.add(randomAlphaOfLength(10)); + } + Map> aliasToIndices = new HashMap<>(); + for (String alias : randomSubsetOf(randomIntBetween(1, 10), indices)) { + Set indicesInAlias; + do { + indicesInAlias = new HashSet<>(randomSubsetOf(randomIntBetween(1, 3), indices)); + indicesInAlias.remove(alias); + } while (indicesInAlias.isEmpty()); + aliasToIndices.put(alias, indicesInAlias); + } + int properAliases = randomIntBetween(0, 3); + for (int i = 0; i < properAliases; i++) { + aliasToIndices.put(randomAlphaOfLength(5), new HashSet<>(randomSubsetOf(randomIntBetween(1, 3), indices))); + } + ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (String index : indices) { + IndexMetadata.Builder indexBuilder = IndexMetadata.builder(index) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0); + aliasToIndices.forEach((key, value) -> { + if (value.contains(index)) { + indexBuilder.putAlias(AliasMetadata.builder(key).build()); + } + }); + projectBuilder.put(indexBuilder); + } + + Exception e = expectThrows(IllegalStateException.class, projectBuilder::build); + assertThat(e.getMessage(), startsWith("index, alias, and data stream names need to be unique")); + } + + public void testValidateAliasWriteOnly() { + String alias = randomAlphaOfLength(5); + String indexA = randomAlphaOfLength(6); + String indexB = randomAlphaOfLength(7); + Boolean aWriteIndex = randomBoolean() ? null : randomBoolean(); + Boolean bWriteIndex; + if (Boolean.TRUE.equals(aWriteIndex)) { + bWriteIndex = randomFrom(Boolean.FALSE, null); + } else { + bWriteIndex = randomFrom(Boolean.TRUE, Boolean.FALSE, null); + } + // when only one index/alias pair exist + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(buildIndexMetadata(indexA, alias, aWriteIndex)) + .build(); + + // when alias points to two indices, but valid + // one of the following combinations: [(null, null), (null, true), (null, false), (false, false)] + ProjectMetadata.builder(project).put(buildIndexMetadata(indexB, alias, bWriteIndex)).build(); + + // when too many write indices + Exception exception = expectThrows(IllegalStateException.class, () -> { + IndexMetadata.Builder metaA = buildIndexMetadata(indexA, alias, true); + IndexMetadata.Builder metaB = buildIndexMetadata(indexB, alias, true); + ProjectMetadata.builder(randomProjectIdOrDefault()).put(metaA).put(metaB).build(); + }); + assertThat(exception.getMessage(), startsWith("alias [" + alias + "] has more than one write index [")); + } + + public void testValidateHiddenAliasConsistency() { + String alias = randomAlphaOfLength(5); + String indexA = randomAlphaOfLength(6); + String indexB = randomAlphaOfLength(7); + + { + Exception ex = expectThrows( + IllegalStateException.class, + () -> buildMetadataWithHiddenIndexMix(alias, indexA, true, indexB, randomFrom(false, null)).build() + ); + assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); + } + + { + Exception ex = expectThrows( + IllegalStateException.class, + () -> buildMetadataWithHiddenIndexMix(alias, indexA, randomFrom(false, null), indexB, true).build() + ); + assertThat(ex.getMessage(), containsString("has is_hidden set to true on indices")); + } + } + + private ProjectMetadata.Builder buildMetadataWithHiddenIndexMix( + String aliasName, + String indexAName, + Boolean indexAHidden, + String indexBName, + Boolean indexBHidden + ) { + IndexMetadata.Builder indexAMeta = IndexMetadata.builder(indexAName) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder(aliasName).isHidden(indexAHidden).build()); + IndexMetadata.Builder indexBMeta = IndexMetadata.builder(indexBName) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder(aliasName).isHidden(indexBHidden).build()); + return ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexAMeta).put(indexBMeta); + } + + public void testResolveIndexRouting() { + IndexMetadata.Builder builder = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias0").build()) + .putAlias(AliasMetadata.builder("alias1").routing("1").build()) + .putAlias(AliasMetadata.builder("alias2").routing("1,2").build()); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(builder).build(); + + // no alias, no index + assertNull(project.resolveIndexRouting(null, null)); + assertEquals(project.resolveIndexRouting("0", null), "0"); + + // index, no alias + assertNull(project.resolveIndexRouting(null, "index")); + assertEquals(project.resolveIndexRouting("0", "index"), "0"); + + // alias with no index routing + assertNull(project.resolveIndexRouting(null, "alias0")); + assertEquals(project.resolveIndexRouting("0", "alias0"), "0"); + + // alias with index routing. + assertEquals(project.resolveIndexRouting(null, "alias1"), "1"); + Exception ex = expectThrows(IllegalArgumentException.class, () -> project.resolveIndexRouting("0", "alias1")); + assertThat( + ex.getMessage(), + is("Alias [alias1] has index routing associated with it [1], and was provided with routing value [0], rejecting operation") + ); + + // alias with invalid index routing. + ex = expectThrows(IllegalArgumentException.class, () -> project.resolveIndexRouting(null, "alias2")); + assertThat( + ex.getMessage(), + is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") + ); + + ex = expectThrows(IllegalArgumentException.class, () -> project.resolveIndexRouting("1", "alias2")); + assertThat( + ex.getMessage(), + is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") + ); + + IndexMetadata.Builder builder2 = IndexMetadata.builder("index2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias0").build()); + ProjectMetadata projectTwoIndices = ProjectMetadata.builder(project).put(builder2).build(); + + // alias with multiple indices + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> projectTwoIndices.resolveIndexRouting("1", "alias0") + ); + assertThat(exception.getMessage(), startsWith("Alias [alias0] has more than one index associated with it")); + } + + public void testResolveWriteIndexRouting() { + AliasMetadata.Builder aliasZeroBuilder = AliasMetadata.builder("alias0"); + if (randomBoolean()) { + aliasZeroBuilder.writeIndex(true); + } + IndexMetadata.Builder builder = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(aliasZeroBuilder.build()) + .putAlias(AliasMetadata.builder("alias1").routing("1").build()) + .putAlias(AliasMetadata.builder("alias2").routing("1,2").build()) + .putAlias(AliasMetadata.builder("alias3").writeIndex(false).build()) + .putAlias(AliasMetadata.builder("alias4").routing("1,2").writeIndex(true).build()); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(builder).build(); + + // no alias, no index + assertNull(project.resolveWriteIndexRouting(null, null)); + assertEquals(project.resolveWriteIndexRouting("0", null), "0"); + + // index, no alias + assertNull(project.resolveWriteIndexRouting(null, "index")); + assertEquals(project.resolveWriteIndexRouting("0", "index"), "0"); + + // alias with no index routing + assertNull(project.resolveWriteIndexRouting(null, "alias0")); + assertEquals(project.resolveWriteIndexRouting("0", "alias0"), "0"); + + // alias with index routing. + assertEquals(project.resolveWriteIndexRouting(null, "alias1"), "1"); + Exception exception = expectThrows(IllegalArgumentException.class, () -> project.resolveWriteIndexRouting("0", "alias1")); + assertThat( + exception.getMessage(), + is("Alias [alias1] has index routing associated with it [1], and was provided with routing value [0], rejecting operation") + ); + + // alias with invalid index routing. + exception = expectThrows(IllegalArgumentException.class, () -> project.resolveWriteIndexRouting(null, "alias2")); + assertThat( + exception.getMessage(), + is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") + ); + exception = expectThrows(IllegalArgumentException.class, () -> project.resolveWriteIndexRouting("1", "alias2")); + assertThat( + exception.getMessage(), + is("index/alias [alias2] provided with routing value [1,2] that resolved to several routing values, rejecting operation") + ); + exception = expectThrows(IllegalArgumentException.class, () -> project.resolveWriteIndexRouting(randomFrom("1", null), "alias4")); + assertThat( + exception.getMessage(), + is("index/alias [alias4] provided with routing value [1,2] that resolved to several routing values, rejecting operation") + ); + + // alias with no write index + exception = expectThrows(IllegalArgumentException.class, () -> project.resolveWriteIndexRouting("1", "alias3")); + assertThat(exception.getMessage(), is("alias [alias3] does not have a write index")); + + // aliases with multiple indices + AliasMetadata.Builder aliasZeroBuilderTwo = AliasMetadata.builder("alias0"); + if (randomBoolean()) { + aliasZeroBuilder.writeIndex(false); + } + IndexMetadata.Builder builder2 = IndexMetadata.builder("index2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(aliasZeroBuilderTwo.build()) + .putAlias(AliasMetadata.builder("alias1").routing("0").writeIndex(true).build()) + .putAlias(AliasMetadata.builder("alias2").writeIndex(true).build()); + ProjectMetadata projectTwoIndices = ProjectMetadata.builder(project).put(builder2).build(); + + // verify that new write index is used + assertThat("0", equalTo(projectTwoIndices.resolveWriteIndexRouting("0", "alias1"))); + } + + public void testFindMappings() throws IOException { + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(IndexMetadata.builder("index1").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(FIND_MAPPINGS_TEST_ITEM)) + .put(IndexMetadata.builder("index2").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(FIND_MAPPINGS_TEST_ITEM)) + .build(); + + { + AtomicInteger onNextIndexCalls = new AtomicInteger(0); + Map mappings = project.findMappings( + Strings.EMPTY_ARRAY, + MapperPlugin.NOOP_FIELD_FILTER, + onNextIndexCalls::incrementAndGet + ); + assertThat(mappings, anEmptyMap()); + assertThat(onNextIndexCalls.get(), equalTo(0)); + } + { + AtomicInteger onNextIndexCalls = new AtomicInteger(0); + Map mappings = project.findMappings( + new String[] { "index1" }, + MapperPlugin.NOOP_FIELD_FILTER, + onNextIndexCalls::incrementAndGet + ); + assertThat(mappings, aMapWithSize(1)); + assertIndexMappingsNotFiltered(mappings, "index1"); + assertThat(onNextIndexCalls.get(), equalTo(1)); + } + { + AtomicInteger onNextIndexCalls = new AtomicInteger(0); + Map mappings = project.findMappings( + new String[] { "index1", "index2" }, + MapperPlugin.NOOP_FIELD_FILTER, + onNextIndexCalls::incrementAndGet + ); + assertThat(mappings, aMapWithSize(2)); + assertIndexMappingsNotFiltered(mappings, "index1"); + assertIndexMappingsNotFiltered(mappings, "index2"); + assertThat(onNextIndexCalls.get(), equalTo(2)); + } + } + + public void testFindMappingsNoOpFilters() throws IOException { + MappingMetadata originalMappingMetadata = new MappingMetadata( + "_doc", + XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, true) + ); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(IndexMetadata.builder("index1").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(originalMappingMetadata)) + .build(); + + { + Map mappings = project.findMappings( + new String[] { "index1" }, + MapperPlugin.NOOP_FIELD_FILTER, + Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP + ); + MappingMetadata mappingMetadata = mappings.get("index1"); + assertSame(originalMappingMetadata, mappingMetadata); + } + { + Map mappings = project.findMappings( + new String[] { "index1" }, + index -> field -> randomBoolean(), + Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP + ); + MappingMetadata mappingMetadata = mappings.get("index1"); + assertNotSame(originalMappingMetadata, mappingMetadata); + } + } + + @SuppressWarnings("unchecked") + public void testFindMappingsWithFilters() throws IOException { + String mapping = FIND_MAPPINGS_TEST_ITEM; + if (randomBoolean()) { + Map stringObjectMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, FIND_MAPPINGS_TEST_ITEM, false); + Map doc = (Map) stringObjectMap.get("_doc"); + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.map(doc); + mapping = Strings.toString(builder); + } + } + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(IndexMetadata.builder("index1").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(mapping)) + .put(IndexMetadata.builder("index2").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(mapping)) + .put(IndexMetadata.builder("index3").settings(indexSettings(IndexVersion.current(), 1, 0)).putMapping(mapping)) + .build(); + + { + Map mappings = project.findMappings(new String[] { "index1", "index2", "index3" }, index -> { + if (index.equals("index1")) { + return field -> field.startsWith("name.") == false + && field.startsWith("properties.key.") == false + && field.equals("age") == false + && field.equals("address.location") == false; + } + if (index.equals("index2")) { + return Predicates.never(); + } + return FieldPredicate.ACCEPT_ALL; + }, Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP); + + assertIndexMappingsNoFields(mappings, "index2"); + assertIndexMappingsNotFiltered(mappings, "index3"); + + MappingMetadata docMapping = mappings.get("index1"); + assertNotNull(docMapping); + + Map sourceAsMap = docMapping.getSourceAsMap(); + assertThat(sourceAsMap.keySet(), containsInAnyOrder("properties", "_routing", "_source")); + + Map typeProperties = (Map) sourceAsMap.get("properties"); + assertThat(typeProperties.keySet(), containsInAnyOrder("name", "address", "birth", "ip", "suggest", "properties")); + + Map name = (Map) typeProperties.get("name"); + assertThat(name.keySet(), containsInAnyOrder("properties")); + Map nameProperties = (Map) name.get("properties"); + assertThat(nameProperties, anEmptyMap()); + + Map address = (Map) typeProperties.get("address"); + assertThat(address.keySet(), containsInAnyOrder("type", "properties")); + Map addressProperties = (Map) address.get("properties"); + assertThat(addressProperties.keySet(), containsInAnyOrder("street", "area")); + assertLeafs(addressProperties, "street", "area"); + + Map properties = (Map) typeProperties.get("properties"); + assertThat(properties.keySet(), containsInAnyOrder("type", "properties")); + Map propertiesProperties = (Map) properties.get("properties"); + assertThat(propertiesProperties.keySet(), containsInAnyOrder("key", "value")); + assertLeafs(propertiesProperties, "key"); + assertMultiField(propertiesProperties, "value", "keyword"); + } + + { + Map mappings = project.findMappings( + new String[] { "index1", "index2", "index3" }, + index -> field -> (index.equals("index3") && field.endsWith("keyword")), + Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP + ); + + assertIndexMappingsNoFields(mappings, "index1"); + assertIndexMappingsNoFields(mappings, "index2"); + MappingMetadata mappingMetadata = mappings.get("index3"); + Map sourceAsMap = mappingMetadata.getSourceAsMap(); + assertThat(sourceAsMap.keySet(), containsInAnyOrder("_routing", "_source", "properties")); + Map typeProperties = (Map) sourceAsMap.get("properties"); + assertThat(typeProperties.keySet(), containsInAnyOrder("properties")); + Map properties = (Map) typeProperties.get("properties"); + assertThat(properties.keySet(), containsInAnyOrder("type", "properties")); + Map propertiesProperties = (Map) properties.get("properties"); + assertThat(propertiesProperties.keySet(), containsInAnyOrder("key", "value")); + Map key = (Map) propertiesProperties.get("key"); + assertThat(key.keySet(), containsInAnyOrder("properties")); + Map keyProperties = (Map) key.get("properties"); + assertThat(keyProperties.keySet(), containsInAnyOrder("keyword")); + assertLeafs(keyProperties, "keyword"); + Map value = (Map) propertiesProperties.get("value"); + assertThat(value.keySet(), containsInAnyOrder("properties")); + Map valueProperties = (Map) value.get("properties"); + assertThat(valueProperties.keySet(), containsInAnyOrder("keyword")); + assertLeafs(valueProperties, "keyword"); + } + + { + Map mappings = project.findMappings( + new String[] { "index1", "index2", "index3" }, + index -> field -> (index.equals("index2")), + Metadata.ON_NEXT_INDEX_FIND_MAPPINGS_NOOP + ); + + assertIndexMappingsNoFields(mappings, "index1"); + assertIndexMappingsNoFields(mappings, "index3"); + assertIndexMappingsNotFiltered(mappings, "index2"); + } + } + + public void testOldestIndexComputation() { + ProjectMetadata project = buildIndicesWithVersions( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersion.current(), + IndexVersion.fromId(IndexVersion.current().id() + 1) + ).build(); + + assertEquals(IndexVersions.MINIMUM_COMPATIBLE, project.oldestIndexVersion()); + + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + assertEquals(IndexVersion.current(), b.build().oldestIndexVersion()); + + Throwable ex = expectThrows( + IllegalArgumentException.class, + () -> buildIndicesWithVersions( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.ZERO, + IndexVersion.fromId(IndexVersion.current().id() + 1) + ).build() + ); + + assertEquals("[index.version.created] is not present in the index settings for index with UUID [null]", ex.getMessage()); + } + + private ProjectMetadata.Builder buildIndicesWithVersions(IndexVersion... indexVersions) { + int lastIndexNum = randomIntBetween(9, 50); + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (IndexVersion indexVersion : indexVersions) { + IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum)) + .settings(settings(indexVersion)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + b.put(im, false); + lastIndexNum = randomIntBetween(lastIndexNum + 1, lastIndexNum + 50); + } + + return b; + } + + private static IndexMetadata.Builder buildIndexMetadata(String name, String alias, Boolean writeIndex) { + return IndexMetadata.builder(name) + .settings(settings(IndexVersion.current())) + .creationDate(randomNonNegativeLong()) + .putAlias(AliasMetadata.builder(alias).writeIndex(writeIndex)) + .numberOfShards(1) + .numberOfReplicas(0); + } + + @SuppressWarnings("unchecked") + private static void assertIndexMappingsNoFields(Map mappings, String index) { + MappingMetadata docMapping = mappings.get(index); + assertNotNull(docMapping); + Map sourceAsMap = docMapping.getSourceAsMap(); + assertThat(sourceAsMap.keySet(), containsInAnyOrder("_routing", "_source", "properties")); + Map typeProperties = (Map) sourceAsMap.get("properties"); + assertThat(typeProperties, anEmptyMap()); + } + + @SuppressWarnings("unchecked") + private static void assertIndexMappingsNotFiltered(Map mappings, String index) { + MappingMetadata docMapping = mappings.get(index); + assertNotNull(docMapping); + + Map sourceAsMap = docMapping.getSourceAsMap(); + assertThat(sourceAsMap.keySet(), containsInAnyOrder("_routing", "_source", "properties")); + + Map typeProperties = (Map) sourceAsMap.get("properties"); + assertThat(typeProperties.keySet(), containsInAnyOrder("name", "address", "birth", "age", "ip", "suggest", "properties")); + + Map name = (Map) typeProperties.get("name"); + assertThat(name.keySet(), containsInAnyOrder("properties")); + Map nameProperties = (Map) name.get("properties"); + assertThat(nameProperties.keySet(), containsInAnyOrder("first", "last")); + assertLeafs(nameProperties, "first", "last"); + + Map address = (Map) typeProperties.get("address"); + assertThat(address.keySet(), containsInAnyOrder("type", "properties")); + Map addressProperties = (Map) address.get("properties"); + assertThat(addressProperties.keySet(), containsInAnyOrder("street", "location", "area")); + assertLeafs(addressProperties, "street", "location", "area"); + + Map properties = (Map) typeProperties.get("properties"); + assertThat(properties.keySet(), containsInAnyOrder("type", "properties")); + Map propertiesProperties = (Map) properties.get("properties"); + assertThat(propertiesProperties.keySet(), containsInAnyOrder("key", "value")); + assertMultiField(propertiesProperties, "key", "keyword"); + assertMultiField(propertiesProperties, "value", "keyword"); + } + + @SuppressWarnings("unchecked") + public static void assertLeafs(Map properties, String... fields) { + assertThat(properties.keySet(), hasItems(fields)); + for (String field : fields) { + Map fieldProp = (Map) properties.get(field); + assertThat(fieldProp, not(hasKey("properties"))); + assertThat(fieldProp, not(hasKey("fields"))); + } + } + + public static void assertMultiField(Map properties, String field, String... subFields) { + assertThat(properties, hasKey(field)); + @SuppressWarnings("unchecked") + Map fieldProp = (Map) properties.get(field); + assertThat(fieldProp, hasKey("fields")); + @SuppressWarnings("unchecked") + Map subFieldsDef = (Map) fieldProp.get("fields"); + assertLeafs(subFieldsDef, subFields); + } + + private static final String FIND_MAPPINGS_TEST_ITEM = """ + { + "_doc": { + "_routing": { + "required":true + }, "_source": { + "enabled":false + }, "properties": { + "name": { + "properties": { + "first": { + "type": "keyword" + }, + "last": { + "type": "keyword" + } + } + }, + "birth": { + "type": "date" + }, + "age": { + "type": "integer" + }, + "ip": { + "type": "ip" + }, + "suggest" : { + "type": "completion" + }, + "address": { + "type": "object", + "properties": { + "street": { + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "area": { + "type": "geo_shape", \s + "tree": "quadtree", + "precision": "1m" + } + } + }, + "properties": { + "type": "nested", + "properties": { + "key" : { + "type": "text", + "fields": { + "keyword" : { + "type" : "keyword" + } + } + }, + "value" : { + "type": "text", + "fields": { + "keyword" : { + "type" : "keyword" + } + } + } + } + } + } + } + } + }"""; + + public void testBuilderRejectsNullCustom() { + final ProjectMetadata.Builder builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + final String key = randomAlphaOfLength(10); + assertThat(expectThrows(NullPointerException.class, () -> builder.putCustom(key, null)).getMessage(), containsString(key)); + } + + public void testBuilderRejectsNullInCustoms() { + final ProjectMetadata.Builder builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + final String key = randomAlphaOfLength(10); + { + final Map map = new HashMap<>(); + map.put(key, null); + assertThat(expectThrows(NullPointerException.class, () -> builder.customs(map)).getMessage(), containsString(key)); + } + } + + public void testCopyAndUpdate() { + var initialIndexUUID = randomUUID(); + final String indexName = randomAlphaOfLengthBetween(4, 12); + final ProjectMetadata before = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(IndexMetadata.builder(indexName).settings(indexSettings(IndexVersion.current(), initialIndexUUID, 1, 1))) + .build(); + + var alteredIndexUUID = randomUUID(); + assertThat(alteredIndexUUID, not(equalTo(initialIndexUUID))); + final ProjectMetadata after = before.copyAndUpdate( + builder -> builder.put(IndexMetadata.builder(indexName).settings(indexSettings(IndexVersion.current(), alteredIndexUUID, 1, 1))) + ); + + assertThat(after, not(sameInstance(before))); + assertThat(after.index(indexName).getIndexUUID(), equalTo(alteredIndexUUID)); + } + + public void testBuilderRemoveCustomIf() { + var custom1 = new TestProjectCustomMetadata(); + var custom2 = new TestProjectCustomMetadata(); + var builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + builder.putCustom("custom1", custom1); + builder.putCustom("custom2", custom2); + + builder.removeCustomIf((key, value) -> Objects.equals(key, "custom1")); + + var project = builder.build(); + assertThat(project.custom("custom1"), nullValue()); + assertThat(project.custom("custom2"), sameInstance(custom2)); + } + + public void testBuilderRejectsDataStreamThatConflictsWithIndex() { + final String dataStreamName = "my-data-stream"; + IndexMetadata idx = createFirstBackingIndex(dataStreamName).build(); + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(idx, false) + .put( + IndexMetadata.builder(dataStreamName) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + .build(), + false + ) + .put(newInstance(dataStreamName, List.of(idx.getIndex()))); + + IllegalStateException e = expectThrows(IllegalStateException.class, b::build); + assertThat( + e.getMessage(), + containsString( + "index, alias, and data stream names need to be unique, but the following duplicates were found [data " + + "stream [" + + dataStreamName + + "] conflicts with index]" + ) + ); + } + + public void testBuilderRejectsDataStreamThatConflictsWithAlias() { + final String dataStreamName = "my-data-stream"; + IndexMetadata idx = createFirstBackingIndex(dataStreamName).putAlias(AliasMetadata.builder(dataStreamName).build()).build(); + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(idx, false) + .put(newInstance(dataStreamName, List.of(idx.getIndex()))); + + IllegalStateException e = expectThrows(IllegalStateException.class, b::build); + assertThat( + e.getMessage(), + containsString( + "index, alias, and data stream names need to be unique, but the following duplicates were found [" + + dataStreamName + + " (alias of [" + + idx.getIndex().getName() + + "]) conflicts with data stream]" + ) + ); + } + + public void testBuilderRejectsAliasThatRefersToDataStreamBackingIndex() { + final String dataStreamName = "my-data-stream"; + final String conflictingName = DataStream.getDefaultBackingIndexName(dataStreamName, 2); + IndexMetadata idx = createFirstBackingIndex(dataStreamName).putAlias(new AliasMetadata.Builder(conflictingName)).build(); + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(idx, false) + .put(newInstance(dataStreamName, List.of(idx.getIndex()))); + + AssertionError e = expectThrows(AssertionError.class, b::build); + assertThat(e.getMessage(), containsString("aliases [" + conflictingName + "] cannot refer to backing indices of data streams")); + } + + public void testBuilderForDataStreamWithRandomlyNumberedBackingIndices() { + final String dataStreamName = "my-data-stream"; + final List backingIndices = new ArrayList<>(); + final int numBackingIndices = randomIntBetween(2, 5); + int lastBackingIndexNum = 0; + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (int k = 1; k <= numBackingIndices; k++) { + lastBackingIndexNum = randomIntBetween(lastBackingIndexNum + 1, lastBackingIndexNum + 50); + IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, lastBackingIndexNum)) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + b.put(im, false); + backingIndices.add(im.getIndex()); + } + + b.put(newInstance(dataStreamName, backingIndices, lastBackingIndexNum, null)); + ProjectMetadata project = b.build(); + assertThat(project.dataStreams().keySet(), containsInAnyOrder(dataStreamName)); + assertThat(project.dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + } + + public void testBuildIndicesLookupForDataStreams() { + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + int numDataStreams = randomIntBetween(2, 8); + for (int i = 0; i < numDataStreams; i++) { + String name = "data-stream-" + i; + addDataStream(name, b); + } + + ProjectMetadata project = b.build(); + assertThat(project.dataStreams().size(), equalTo(numDataStreams)); + for (int i = 0; i < numDataStreams; i++) { + String name = "data-stream-" + i; + IndexAbstraction value = project.getIndicesLookup().get(name); + assertThat(value, notNullValue()); + DataStream ds = project.dataStreams().get(name); + assertThat(ds, notNullValue()); + + assertThat(value.isHidden(), is(false)); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + assertThat(value.getIndices(), hasSize(ds.getIndices().size())); + assertThat(value.getWriteIndex().getName(), DataStreamTestHelper.backingIndexEqualTo(name, (int) ds.getGeneration())); + } + } + + public void testBuildIndicesLookupForDataStreamAliases() { + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + + addDataStream("d1", b); + addDataStream("d2", b); + addDataStream("d3", b); + addDataStream("d4", b); + + b.put("a1", "d1", null, null); + b.put("a1", "d2", null, null); + b.put("a2", "d3", null, null); + b.put("a3", "d1", null, null); + + ProjectMetadata project = b.build(); + assertThat(project.dataStreams(), aMapWithSize(4)); + IndexAbstraction value = project.getIndicesLookup().get("d1"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + + value = project.getIndicesLookup().get("d2"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + + value = project.getIndicesLookup().get("d3"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + + value = project.getIndicesLookup().get("d4"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); + + value = project.getIndicesLookup().get("a1"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.ALIAS)); + + value = project.getIndicesLookup().get("a2"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.ALIAS)); + + value = project.getIndicesLookup().get("a3"); + assertThat(value, notNullValue()); + assertThat(value.getType(), equalTo(IndexAbstraction.Type.ALIAS)); + } + + public void testDataStreamAliasValidation() { + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + addDataStream("my-alias", b); + b.put("my-alias", "my-alias", null, null); + var e = expectThrows(IllegalStateException.class, b::build); + assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); + + b = ProjectMetadata.builder(randomProjectIdOrDefault()); + addDataStream("d1", b); + addDataStream("my-alias", b); + b.put("my-alias", "d1", null, null); + e = expectThrows(IllegalStateException.class, b::build); + assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); + + b = ProjectMetadata.builder(randomProjectIdOrDefault()); + b.put( + IndexMetadata.builder("index1") + .settings(indexSettings(IndexVersion.current(), 1, 0)) + .putAlias(new AliasMetadata.Builder("my-alias")) + ); + + addDataStream("d1", b); + b.put("my-alias", "d1", null, null); + e = expectThrows(IllegalStateException.class, b::build); + assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)")); + } + + public void testDataStreamAliasValidationRestoreScenario() { + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + b.dataStreams( + Map.of("my-alias", createDataStream("my-alias")), + Map.of("my-alias", new DataStreamAlias("my-alias", List.of("my-alias"), null, null)) + ); + var e = expectThrows(IllegalStateException.class, b::build); + assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); + + b = ProjectMetadata.builder(randomProjectIdOrDefault()); + b.dataStreams( + Map.of("d1", createDataStream("d1"), "my-alias", createDataStream("my-alias")), + Map.of("my-alias", new DataStreamAlias("my-alias", List.of("d1"), null, null)) + ); + e = expectThrows(IllegalStateException.class, b::build); + assertThat(e.getMessage(), containsString("data stream alias and data stream have the same name (my-alias)")); + + b = ProjectMetadata.builder(randomProjectIdOrDefault()); + b.put( + IndexMetadata.builder("index1") + .settings(indexSettings(IndexVersion.current(), 1, 0)) + .putAlias(new AliasMetadata.Builder("my-alias")) + ); + b.dataStreams(Map.of("d1", createDataStream("d1")), Map.of("my-alias", new DataStreamAlias("my-alias", List.of("d1"), null, null))); + e = expectThrows(IllegalStateException.class, b::build); + assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)")); + } + + private void addDataStream(String name, ProjectMetadata.Builder b) { + int numBackingIndices = randomIntBetween(1, 4); + List indices = new ArrayList<>(numBackingIndices); + for (int j = 1; j <= numBackingIndices; j++) { + IndexMetadata idx = createBackingIndex(name, j).build(); + indices.add(idx.getIndex()); + b.put(idx, true); + } + b.put(newInstance(name, indices)); + } + + private DataStream createDataStream(String name) { + int numBackingIndices = randomIntBetween(1, 4); + List indices = new ArrayList<>(numBackingIndices); + for (int j = 1; j <= numBackingIndices; j++) { + IndexMetadata idx = createBackingIndex(name, j).build(); + indices.add(idx.getIndex()); + } + return newInstance(name, indices); + } + + public void testIndicesLookupRecordsDataStreamForBackingIndices() { + final int numIndices = randomIntBetween(2, 5); + final int numBackingIndices = randomIntBetween(2, 5); + final String dataStreamName = "my-data-stream"; + CreateIndexResult result = createIndices(numIndices, numBackingIndices, dataStreamName); + + SortedMap indicesLookup = result.project.getIndicesLookup(); + assertThat(indicesLookup, aMapWithSize(result.indices.size() + result.backingIndices.size() + 1)); + for (Index index : result.indices) { + assertThat(indicesLookup, hasKey(index.getName())); + assertNull(indicesLookup.get(index.getName()).getParentDataStream()); + } + for (Index index : result.backingIndices) { + assertThat(indicesLookup, hasKey(index.getName())); + assertNotNull(indicesLookup.get(index.getName()).getParentDataStream()); + assertThat(indicesLookup.get(index.getName()).getParentDataStream().getName(), equalTo(dataStreamName)); + } + } + + public void testValidateDataStreamsNoConflicts() { + ProjectMetadata project = createIndices(5, 10, "foo-datastream").project; + // don't expect any exception when validating a system without indices that would conflict with future backing indices + assertDataStreams(project.indices(), (DataStreamMetadata) project.customs().get(DataStreamMetadata.TYPE)); + } + + public void testValidateDataStreamsIgnoresIndicesWithoutCounter() { + String dataStreamName = "foo-datastream"; + ProjectMetadata project = ProjectMetadata.builder(createIndices(10, 10, dataStreamName).project) + .put( + new IndexMetadata.Builder(dataStreamName + "-index-without-counter").settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + ) + .put( + new IndexMetadata.Builder(dataStreamName + randomAlphaOfLength(10)).settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + + ) + .put( + new IndexMetadata.Builder(randomAlphaOfLength(10)).settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + + ) + .build(); + // don't expect any exception when validating against non-backing indices that don't conform to the backing indices naming + // convention + assertDataStreams(project.indices(), (DataStreamMetadata) project.customs().get(DataStreamMetadata.TYPE)); + } + + public void testValidateDataStreamsAllowsNamesThatStartsWithPrefix() { + String dataStreamName = "foo-datastream"; + ProjectMetadata project = ProjectMetadata.builder(createIndices(10, 10, dataStreamName).project) + .put( + new IndexMetadata.Builder(DataStream.BACKING_INDEX_PREFIX + dataStreamName + "-something-100012").settings( + settings(IndexVersion.current()) + ).numberOfShards(1).numberOfReplicas(1) + ) + .build(); + // don't expect any exception when validating against (potentially backing) indices that can't create conflict because of + // additional text before number + assertDataStreams(project.indices(), (DataStreamMetadata) project.customs().get(DataStreamMetadata.TYPE)); + } + + public void testValidateDataStreamsForNullDataStreamMetadata() { + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put(IndexMetadata.builder("foo-index").settings(settings(IndexVersion.current())).numberOfShards(1).numberOfReplicas(1)) + .build(); + + try { + assertDataStreams(project.indices(), DataStreamMetadata.EMPTY); + } catch (Exception e) { + fail("did not expect exception when validating a system without any data streams but got " + e.getMessage()); + } + } + + public void testDataStreamAliases() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null), is(true)); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-us", null, null), is(true)); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au", null, null), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-au", null, null), is(false)); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") + ); + } + + public void testDataStreamReferToNonExistingDataStream() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null)); + assertThat(e.getMessage(), equalTo("alias [logs-postgres] refers to a non existing data stream [logs-postgres-eu]")); + + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + } + + public void testDeleteDataStreamShouldUpdateAlias() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us", null, null); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au", null, null); + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") + ); + + mdBuilder = ProjectMetadata.builder(project); + mdBuilder.removeDataStream("logs-postgres-us"); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-au") + ); + + mdBuilder = ProjectMetadata.builder(project); + mdBuilder.removeDataStream("logs-postgres-au"); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + + mdBuilder = ProjectMetadata.builder(project); + mdBuilder.removeDataStream("logs-postgres-eu"); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), nullValue()); + } + + public void testDeleteDataStreamAlias() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us", null, null); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au", null, null); + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") + ); + + mdBuilder = ProjectMetadata.builder(project); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-us", true), is(true)); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-au") + ); + + mdBuilder = ProjectMetadata.builder(project); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-au", true), is(true)); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-eu")); + + mdBuilder = ProjectMetadata.builder(project); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-eu", true), is(true)); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), nullValue()); + } + + public void testDeleteDataStreamAliasMustExists() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-eu")); + mdBuilder.put("logs-postgres", "logs-postgres-eu", null, null); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-us")); + mdBuilder.put("logs-postgres", "logs-postgres-us", null, null); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-au")); + mdBuilder.put("logs-postgres", "logs-postgres-au", null, null); + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-eu", "logs-postgres-us", "logs-postgres-au") + ); + + ProjectMetadata.Builder mdBuilder2 = ProjectMetadata.builder(project); + expectThrows(ResourceNotFoundException.class, () -> mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", true)); + assertThat(mdBuilder2.removeDataStreamAlias("logs-mysql", "logs-postgres-us", false), is(false)); + } + + public void testDataStreamWriteAlias() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); + mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); + + mdBuilder = ProjectMetadata.builder(project); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", true, null), is(true)); + + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-replicated")); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); + } + + public void testDataStreamMultipleWriteAlias() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-foobar")); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-barbaz")); + mdBuilder.put("logs", "logs-foobar", true, null); + mdBuilder.put("logs", "logs-barbaz", true, null); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs").getWriteDataStream(), equalTo("logs-barbaz")); + assertThat(project.dataStreamAliases().get("logs").getDataStreams(), containsInAnyOrder("logs-foobar", "logs-barbaz")); + } + + public void testDataStreamWriteAliasUnset() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); + mdBuilder.put("logs-postgres", "logs-postgres-replicated", true, null); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-replicated")); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); + + mdBuilder = ProjectMetadata.builder(project); + // Side check: null value isn't changing anything: + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(false)); + // Unset write flag + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", false, null), is(true)); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); + } + + public void testDataStreamWriteAliasChange() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-primary")); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", true, null), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(true)); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-primary")); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") + ); + + // change write flag: + mdBuilder = ProjectMetadata.builder(project); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", false, null), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", true, null), is(true)); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-replicated")); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") + ); + } + + public void testDataStreamWriteRemoveAlias() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-primary")); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", true, null), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(true)); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-primary")); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") + ); + + mdBuilder = ProjectMetadata.builder(project); + assertThat(mdBuilder.removeDataStreamAlias("logs-postgres", "logs-postgres-primary", randomBoolean()), is(true)); + project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); + } + + public void testDataStreamWriteRemoveDataStream() { + ProjectMetadata.Builder mdBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-primary")); + mdBuilder.put(DataStreamTestHelper.randomInstance("logs-postgres-replicated")); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-primary", true, null), is(true)); + assertThat(mdBuilder.put("logs-postgres", "logs-postgres-replicated", null, null), is(true)); + + ProjectMetadata project = mdBuilder.build(); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), equalTo("logs-postgres-primary")); + assertThat( + project.dataStreamAliases().get("logs-postgres").getDataStreams(), + containsInAnyOrder("logs-postgres-primary", "logs-postgres-replicated") + ); + + mdBuilder = ProjectMetadata.builder(project); + mdBuilder.removeDataStream("logs-postgres-primary"); + project = mdBuilder.build(); + assertThat(project.dataStreams().keySet(), contains("logs-postgres-replicated")); + assertThat(project.dataStreamAliases().get("logs-postgres"), notNullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getWriteDataStream(), nullValue()); + assertThat(project.dataStreamAliases().get("logs-postgres").getDataStreams(), containsInAnyOrder("logs-postgres-replicated")); + } + + public void testReuseIndicesLookup() { + String indexName = "my-index"; + String aliasName = "my-alias"; + String dataStreamName = "logs-mysql-prod"; + String dataStreamAliasName = "logs-mysql"; + ProjectMetadata previous = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + + // Things that should change indices lookup + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + IndexMetadata idx = DataStreamTestHelper.createFirstBackingIndex(dataStreamName).build(); + builder.put(idx, true); + DataStream dataStream = newInstance(dataStreamName, List.of(idx.getIndex())); + builder.put(dataStream); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + builder.put(dataStreamAliasName, dataStreamName, false, null); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + builder.put(dataStreamAliasName, dataStreamName, true, null); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + builder.put( + IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current())) + .creationDate(randomNonNegativeLong()) + .numberOfShards(1) + .numberOfReplicas(0) + ); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + imBuilder.putAlias(AliasMetadata.builder(aliasName).build()); + builder.put(imBuilder); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + imBuilder.putAlias(AliasMetadata.builder(aliasName).writeIndex(true).build()); + builder.put(imBuilder); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + Settings.Builder sBuilder = Settings.builder() + .put(builder.get(indexName).getSettings()) + .put(IndexMetadata.INDEX_HIDDEN_SETTING.getKey(), true); + imBuilder.settings(sBuilder.build()); + builder.put(imBuilder); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), not(sameInstance(project.getIndicesLookup()))); + previous = project; + } + + // Things that shouldn't change indices lookup + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + imBuilder.numberOfReplicas(2); + builder.put(imBuilder); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), sameInstance(project.getIndicesLookup())); + previous = project; + } + { + ProjectMetadata.Builder builder = ProjectMetadata.builder(previous); + IndexMetadata.Builder imBuilder = IndexMetadata.builder(builder.get(indexName)); + Settings.Builder sBuilder = Settings.builder() + .put(builder.get(indexName).getSettings()) + .put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), "val"); + imBuilder.settings(sBuilder.build()); + builder.put(imBuilder); + ProjectMetadata project = builder.build(); + assertThat(previous.getIndicesLookup(), sameInstance(project.getIndicesLookup())); + previous = project; + } + } + + public void testAliasedIndices() { + int numAliases = randomIntBetween(32, 64); + int numIndicesPerAlias = randomIntBetween(8, 16); + + ProjectMetadata.Builder builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (int i = 0; i < numAliases; i++) { + String aliasName = "alias-" + i; + for (int j = 0; j < numIndicesPerAlias; j++) { + AliasMetadata.Builder alias = new AliasMetadata.Builder(aliasName); + if (j == 0) { + alias.writeIndex(true); + } + + String indexName = aliasName + "-" + j; + builder.put( + IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current())) + .creationDate(randomNonNegativeLong()) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(alias) + ); + } + } + + ProjectMetadata project = builder.build(); + for (int i = 0; i < numAliases; i++) { + String aliasName = "alias-" + i; + Set result = project.aliasedIndices(aliasName); + Index[] expected = IntStream.range(0, numIndicesPerAlias) + .mapToObj(j -> aliasName + "-" + j) + .map(name -> new Index(name, ClusterState.UNKNOWN_UUID)) + .toArray(Index[]::new); + assertThat(result, containsInAnyOrder(expected)); + } + + // Add a new alias and index + builder = ProjectMetadata.builder(project); + String newAliasName = "alias-new"; + { + builder.put( + IndexMetadata.builder(newAliasName + "-1") + .settings(settings(IndexVersion.current())) + .creationDate(randomNonNegativeLong()) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(new AliasMetadata.Builder(newAliasName).writeIndex(true)) + ); + } + project = builder.build(); + assertThat(project.aliasedIndices(), hasSize(numAliases + 1)); + assertThat(project.aliasedIndices(newAliasName), contains(new Index(newAliasName + "-1", ClusterState.UNKNOWN_UUID))); + + // Remove the new alias/index + builder = ProjectMetadata.builder(project); + { + builder.remove(newAliasName + "-1"); + } + project = builder.build(); + assertThat(project.aliasedIndices(), hasSize(numAliases)); + assertThat(project.aliasedIndices(newAliasName), empty()); + + // Add a new alias that points to existing indices + builder = ProjectMetadata.builder(project); + { + IndexMetadata.Builder imBuilder = new IndexMetadata.Builder(project.index("alias-1-0")); + imBuilder.putAlias(new AliasMetadata.Builder(newAliasName)); + builder.put(imBuilder); + + imBuilder = new IndexMetadata.Builder(project.index("alias-2-1")); + imBuilder.putAlias(new AliasMetadata.Builder(newAliasName)); + builder.put(imBuilder); + + imBuilder = new IndexMetadata.Builder(project.index("alias-3-2")); + imBuilder.putAlias(new AliasMetadata.Builder(newAliasName)); + builder.put(imBuilder); + } + project = builder.build(); + assertThat(project.aliasedIndices(), hasSize(numAliases + 1)); + assertThat( + project.aliasedIndices(newAliasName), + containsInAnyOrder( + new Index("alias-1-0", ClusterState.UNKNOWN_UUID), + new Index("alias-2-1", ClusterState.UNKNOWN_UUID), + new Index("alias-3-2", ClusterState.UNKNOWN_UUID) + ) + ); + + // Remove the new alias that points to existing indices + builder = ProjectMetadata.builder(project); + { + IndexMetadata.Builder imBuilder = new IndexMetadata.Builder(project.index("alias-1-0")); + imBuilder.removeAlias(newAliasName); + builder.put(imBuilder); + + imBuilder = new IndexMetadata.Builder(project.index("alias-2-1")); + imBuilder.removeAlias(newAliasName); + builder.put(imBuilder); + + imBuilder = new IndexMetadata.Builder(project.index("alias-3-2")); + imBuilder.removeAlias(newAliasName); + builder.put(imBuilder); + } + project = builder.build(); + assertThat(project.aliasedIndices(), hasSize(numAliases)); + assertThat(project.aliasedIndices(newAliasName), empty()); + } + + public void testHiddenAliasValidation() { + final String hiddenAliasName = "hidden_alias"; + + IndexMetadata hidden1 = buildIndexWithAlias("hidden1", hiddenAliasName, true, IndexVersion.current(), false); + IndexMetadata hidden2 = buildIndexWithAlias("hidden2", hiddenAliasName, true, IndexVersion.current(), false); + IndexMetadata hidden3 = buildIndexWithAlias("hidden3", hiddenAliasName, true, IndexVersion.current(), false); + + IndexMetadata nonHidden = buildIndexWithAlias("nonhidden1", hiddenAliasName, false, IndexVersion.current(), false); + IndexMetadata unspecified = buildIndexWithAlias("nonhidden2", hiddenAliasName, null, IndexVersion.current(), false); + + { + // Should be ok: + projectWithIndices(hidden1, hidden2, hidden3); + } + + { + // Should be ok: + if (randomBoolean()) { + projectWithIndices(nonHidden, unspecified); + } else { + projectWithIndices(unspecified, nonHidden); + } + } + + { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> projectWithIndices(hidden1, hidden2, hidden3, nonHidden) + ); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] has is_hidden set to true on indices [")); + assertThat( + exception.getMessage(), + allOf( + containsString(hidden1.getIndex().getName()), + containsString(hidden2.getIndex().getName()), + containsString(hidden3.getIndex().getName()) + ) + ); + assertThat( + exception.getMessage(), + containsString( + "but does not have is_hidden set to true on indices [" + + nonHidden.getIndex().getName() + + "]; alias must have the same is_hidden setting on all indices" + ) + ); + } + + { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> projectWithIndices(hidden1, hidden2, hidden3, unspecified) + ); + assertThat(exception.getMessage(), containsString("alias [" + hiddenAliasName + "] has is_hidden set to true on indices [")); + assertThat( + exception.getMessage(), + allOf( + containsString(hidden1.getIndex().getName()), + containsString(hidden2.getIndex().getName()), + containsString(hidden3.getIndex().getName()) + ) + ); + assertThat( + exception.getMessage(), + containsString( + "but does not have is_hidden set to true on indices [" + + unspecified.getIndex().getName() + + "]; alias must have the same is_hidden setting on all indices" + ) + ); + } + + { + final IndexMetadata hiddenIndex = randomFrom(hidden1, hidden2, hidden3); + IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { + if (randomBoolean()) { + projectWithIndices(nonHidden, unspecified, hiddenIndex); + } else { + projectWithIndices(unspecified, nonHidden, hiddenIndex); + } + }); + assertThat( + exception.getMessage(), + containsString( + "alias [" + + hiddenAliasName + + "] has is_hidden set to true on " + + "indices [" + + hiddenIndex.getIndex().getName() + + "] but does not have is_hidden set to true on indices [" + ) + ); + assertThat( + exception.getMessage(), + allOf(containsString(unspecified.getIndex().getName()), containsString(nonHidden.getIndex().getName())) + ); + assertThat(exception.getMessage(), containsString("but does not have is_hidden set to true on indices [")); + } + } + + public static final String SYSTEM_ALIAS_NAME = "system_alias"; + + public void testSystemAliasValidationMixedVersionSystemAndRegularFails() { + final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); + final IndexMetadata regularIndex = buildIndexWithAlias("regular1", SYSTEM_ALIAS_NAME, false, IndexVersion.current(), false); + + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> projectWithIndices(currentVersionSystem, oldVersionSystem, regularIndex) + ); + assertThat( + exception.getMessage(), + containsString( + "alias [" + + SYSTEM_ALIAS_NAME + + "] refers to both system indices [" + + currentVersionSystem.getIndex().getName() + + "] and non-system indices: [" + + regularIndex.getIndex().getName() + + "], but aliases must refer to either system or non-system indices, not both" + ) + ); + } + + public void testSystemAliasValidationNewSystemAndRegularFails() { + final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata regularIndex = buildIndexWithAlias("regular1", SYSTEM_ALIAS_NAME, false, IndexVersion.current(), false); + + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> projectWithIndices(currentVersionSystem, regularIndex) + ); + assertThat( + exception.getMessage(), + containsString( + "alias [" + + SYSTEM_ALIAS_NAME + + "] refers to both system indices [" + + currentVersionSystem.getIndex().getName() + + "] and non-system indices: [" + + regularIndex.getIndex().getName() + + "], but aliases must refer to either system or non-system indices, not both" + ) + ); + } + + public void testSystemAliasOldSystemAndNewRegular() { + final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); + final IndexMetadata regularIndex = buildIndexWithAlias("regular1", SYSTEM_ALIAS_NAME, false, IndexVersion.current(), false); + + // Should be ok: + projectWithIndices(oldVersionSystem, regularIndex); + } + + public void testSystemIndexValidationAllRegular() { + final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata currentVersionSystem2 = buildIndexWithAlias(".system2", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); + + // Should be ok + projectWithIndices(currentVersionSystem, currentVersionSystem2, oldVersionSystem); + } + + public void testSystemAliasValidationAllSystemSomeOld() { + final IndexVersion random7xVersion = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata currentVersionSystem2 = buildIndexWithAlias(".system2", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata oldVersionSystem = buildIndexWithAlias(".oldVersionSystem", SYSTEM_ALIAS_NAME, null, random7xVersion, true); + + // Should be ok: + projectWithIndices(currentVersionSystem, currentVersionSystem2, oldVersionSystem); + } + + public void testSystemAliasValidationAll8x() { + final IndexMetadata currentVersionSystem = buildIndexWithAlias(".system1", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + final IndexMetadata currentVersionSystem2 = buildIndexWithAlias(".system2", SYSTEM_ALIAS_NAME, null, IndexVersion.current(), true); + + // Should be ok + projectWithIndices(currentVersionSystem, currentVersionSystem2); + } + + private void projectWithIndices(IndexMetadata... indices) { + ProjectMetadata.Builder builder = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (var cursor : indices) { + builder.put(cursor, false); + } + builder.build(); + } + + private IndexMetadata buildIndexWithAlias( + String indexName, + String aliasName, + @Nullable Boolean aliasIsHidden, + IndexVersion indexCreationVersion, + boolean isSystem + ) { + final AliasMetadata.Builder aliasMetadata = new AliasMetadata.Builder(aliasName); + if (aliasIsHidden != null || randomBoolean()) { + aliasMetadata.isHidden(aliasIsHidden); + } + return new IndexMetadata.Builder(indexName).settings(settings(indexCreationVersion)) + .system(isSystem) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(aliasMetadata) + .build(); + } + + public void testMappingDuplication() { + final Set randomMappingDefinitions; + { + int numEntries = randomIntBetween(4, 8); + randomMappingDefinitions = Sets.newHashSetWithExpectedSize(numEntries); + for (int i = 0; i < numEntries; i++) { + Map mapping = RandomAliasActionsGenerator.randomMap(2); + String mappingAsString = Strings.toString((builder, params) -> builder.mapContents(mapping)); + randomMappingDefinitions.add(mappingAsString); + } + } + + ProjectMetadata project; + int numIndices = randomIntBetween(16, 32); + { + String[] definitions = randomMappingDefinitions.toArray(String[]::new); + ProjectMetadata.Builder mb = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (int i = 0; i < numIndices; i++) { + IndexMetadata.Builder indexBuilder = IndexMetadata.builder("index-" + i) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .putMapping(definitions[i % randomMappingDefinitions.size()]) + .numberOfShards(1) + .numberOfReplicas(0); + if (randomBoolean()) { + mb.put(indexBuilder); + } else { + mb.put(indexBuilder.build(), true); + } + } + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); + assertThat( + project.indices().values().stream().map(IndexMetadata::mapping).collect(Collectors.toSet()), + hasSize(project.getMappingsByHash().size()) + ); + + // Add a new index with a new index with known mapping: + MappingMetadata mapping = project.indices().get("index-" + randomInt(numIndices - 1)).mapping(); + MappingMetadata entry = project.getMappingsByHash().get(mapping.getSha256()); + { + ProjectMetadata.Builder mb = new ProjectMetadata.Builder(project); + mb.put( + IndexMetadata.builder("index-" + numIndices) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .putMapping(mapping) + .numberOfShards(1) + .numberOfReplicas(0) + ); + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); + assertThat(project.getMappingsByHash().get(mapping.getSha256()), equalTo(entry)); + + // Remove index and ensure mapping cache stays the same + { + ProjectMetadata.Builder mb = new ProjectMetadata.Builder(project); + mb.remove("index-" + numIndices); + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); + assertThat(project.getMappingsByHash().get(mapping.getSha256()), equalTo(entry)); + + // Update a mapping of an index: + IndexMetadata luckyIndex = project.index("index-" + randomInt(numIndices - 1)); + entry = project.getMappingsByHash().get(luckyIndex.mapping().getSha256()); + MappingMetadata updatedMapping = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Map.of("mapping", "updated")); + { + ProjectMetadata.Builder mb = new ProjectMetadata.Builder(project); + mb.put(IndexMetadata.builder(luckyIndex).putMapping(updatedMapping)); + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size() + 1)); + assertThat(project.getMappingsByHash().get(luckyIndex.mapping().getSha256()), equalTo(entry)); + assertThat(project.getMappingsByHash().get(updatedMapping.getSha256()), equalTo(updatedMapping)); + + // Remove the index with updated mapping + { + ProjectMetadata.Builder mb = new ProjectMetadata.Builder(project); + mb.remove(luckyIndex.getIndex().getName()); + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); + assertThat(project.getMappingsByHash().get(updatedMapping.getSha256()), nullValue()); + + // Add an index with new mapping and then later remove it: + MappingMetadata newMapping = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, Map.of("new", "mapping")); + { + ProjectMetadata.Builder mb = new ProjectMetadata.Builder(project); + mb.put( + IndexMetadata.builder("index-" + numIndices) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .putMapping(newMapping) + .numberOfShards(1) + .numberOfReplicas(0) + ); + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size() + 1)); + assertThat(project.getMappingsByHash().get(newMapping.getSha256()), equalTo(newMapping)); + + { + ProjectMetadata.Builder mb = new ProjectMetadata.Builder(project); + mb.remove("index-" + numIndices); + project = mb.build(); + } + assertThat(project.getMappingsByHash(), aMapWithSize(randomMappingDefinitions.size())); + assertThat(project.getMappingsByHash().get(newMapping.getSha256()), nullValue()); + } + + public void testWithLifecycleState() { + String indexName = "my-index"; + String indexUUID = randomAlphaOfLength(10); + ProjectMetadata project1 = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put( + IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current()).put(IndexMetadata.SETTING_INDEX_UUID, indexUUID)) + .creationDate(randomNonNegativeLong()) + .numberOfShards(1) + .numberOfReplicas(0) + ) + .build(); + IndexMetadata index1 = project1.index(indexName); + assertThat(project1.getIndicesLookup(), notNullValue()); + assertThat(index1.getLifecycleExecutionState(), sameInstance(LifecycleExecutionState.EMPTY_STATE)); + + LifecycleExecutionState state = LifecycleExecutionState.builder().setPhase("phase").setAction("action").setStep("step").build(); + ProjectMetadata project2 = project1.withLifecycleState(index1.getIndex(), state); + IndexMetadata index2 = project2.index(indexName); + + // the indices lookups are the same object + assertThat(project2.getIndicesLookup(), sameInstance(project1.getIndicesLookup())); + + // the lifecycle state and version were changed + assertThat(index2.getLifecycleExecutionState().asMap(), is(state.asMap())); + assertThat(index2.getVersion(), is(index1.getVersion() + 1)); + + // but those are the only differences between the two + IndexMetadata.Builder builder = IndexMetadata.builder(index2); + builder.version(builder.version() - 1); + builder.removeCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY); + assertThat(index1, equalTo(builder.build())); + + // withLifecycleState returns the same reference if nothing changed + ProjectMetadata project3 = project2.withLifecycleState(index2.getIndex(), state); + assertThat(project3, sameInstance(project2)); + + // withLifecycleState rejects a nonsense Index + String randomUUID = randomValueOtherThan(indexUUID, () -> randomAlphaOfLength(10)); + expectThrows(IndexNotFoundException.class, () -> project1.withLifecycleState(new Index(indexName, randomUUID), state)); + } + + public void testRetrieveIndexModeFromTemplateTsdb() throws IOException { + // tsdb: + var tsdbTemplate = new Template(Settings.builder().put("index.mode", "time_series").build(), new CompressedXContent("{}"), null); + // Settings in component template: + { + var componentTemplate = new ComponentTemplate(tsdbTemplate, null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + ProjectMetadata p = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put("component_template_1", componentTemplate) + .put("index_template_1", indexTemplate) + .build(); + assertThat(p.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES)); + } + // Settings in composable index template: + { + var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(tsdbTemplate) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + ProjectMetadata p = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put("component_template_1", componentTemplate) + .put("index_template_1", indexTemplate) + .build(); + assertThat(p.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.TIME_SERIES)); + } + } + + public void testRetrieveIndexModeFromTemplateLogsdb() throws IOException { + // logsdb: + var logsdbTemplate = new Template(Settings.builder().put("index.mode", "logsdb").build(), new CompressedXContent("{}"), null); + // Settings in component template: + { + var componentTemplate = new ComponentTemplate(logsdbTemplate, null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + ProjectMetadata p = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put("component_template_1", componentTemplate) + .put("index_template_1", indexTemplate) + .build(); + assertThat(p.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB)); + } + // Settings in composable index template: + { + var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(logsdbTemplate) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + ProjectMetadata p = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put("component_template_1", componentTemplate) + .put("index_template_1", indexTemplate) + .build(); + assertThat(p.retrieveIndexModeFromTemplate(indexTemplate), is(IndexMode.LOGSDB)); + } + } + + public void testRetrieveIndexModeFromTemplateEmpty() throws IOException { + // no index mode: + var emptyTemplate = new Template(Settings.EMPTY, new CompressedXContent("{}"), null); + // Settings in component template: + { + var componentTemplate = new ComponentTemplate(emptyTemplate, null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + ProjectMetadata p = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put("component_template_1", componentTemplate) + .put("index_template_1", indexTemplate) + .build(); + assertThat(p.retrieveIndexModeFromTemplate(indexTemplate), nullValue()); + } + // Settings in composable index template: + { + var componentTemplate = new ComponentTemplate(new Template(null, null, null), null, null); + var indexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("test-*")) + .template(emptyTemplate) + .componentTemplates(List.of("component_template_1")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build(); + ProjectMetadata p = ProjectMetadata.builder(randomProjectIdOrDefault()) + .put("component_template_1", componentTemplate) + .put("index_template_1", indexTemplate) + .build(); + assertThat(p.retrieveIndexModeFromTemplate(indexTemplate), nullValue()); + } + } + + private static CreateIndexResult createIndices(int numIndices, int numBackingIndices, String dataStreamName) { + // create some indices that do not back a data stream + final List indices = new ArrayList<>(); + int lastIndexNum = randomIntBetween(9, 50); + ProjectMetadata.Builder b = ProjectMetadata.builder(randomProjectIdOrDefault()); + for (int k = 1; k <= numIndices; k++) { + IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum)) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + b.put(im, false); + indices.add(im.getIndex()); + lastIndexNum = randomIntBetween(lastIndexNum + 1, lastIndexNum + 50); + } + + // create some backing indices for a data stream + final List backingIndices = new ArrayList<>(); + int lastBackingIndexNum = 0; + for (int k = 1; k <= numBackingIndices; k++) { + lastBackingIndexNum = randomIntBetween(lastBackingIndexNum + 1, lastBackingIndexNum + 50); + IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, lastBackingIndexNum)) + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + b.put(im, false); + backingIndices.add(im.getIndex()); + } + b.put(newInstance(dataStreamName, backingIndices, lastBackingIndexNum, null)); + return new CreateIndexResult(indices, backingIndices, b.build()); + } + + private record CreateIndexResult(List indices, List backingIndices, ProjectMetadata project) {}; public void testToXContent() throws IOException { final ProjectMetadata projectMetadata = prepareProjectMetadata(); @@ -670,21 +2857,37 @@ static int expectedChunkCount(ToXContent.Params params, ProjectMetadata project) return Math.toIntExact(chunkCount); } - public void testCopyAndUpdate() { - var initialIndexUUID = randomUUID(); - final String indexName = randomAlphaOfLengthBetween(4, 12); - final ProjectMetadata before = ProjectMetadata.builder(randomProjectIdOrDefault()) - .put(IndexMetadata.builder(indexName).settings(indexSettings(IndexVersion.current(), initialIndexUUID, 1, 1))) - .build(); + private static class TestProjectCustomMetadata implements Metadata.ProjectCustom { - var alteredIndexUUID = randomUUID(); - assertThat(alteredIndexUUID, not(equalTo(initialIndexUUID))); - final ProjectMetadata after = before.copyAndUpdate( - builder -> builder.put(IndexMetadata.builder(indexName).settings(indexSettings(IndexVersion.current(), alteredIndexUUID, 1, 1))) - ); + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Collections.emptyIterator(); + } - assertThat(after, not(sameInstance(before))); - assertThat(after.index(indexName).getIndexUUID(), equalTo(alteredIndexUUID)); + @Override + public Diff diff(Metadata.ProjectCustom previousState) { + return null; + } + + @Override + public EnumSet context() { + return null; + } + + @Override + public String getWriteableName() { + return null; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + + } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index c17c0d10410fa..9fa0760e2ae2a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -35,8 +35,8 @@ import java.util.Set; import java.util.function.Function; -import static org.elasticsearch.cluster.metadata.MetadataTests.assertLeafs; -import static org.elasticsearch.cluster.metadata.MetadataTests.assertMultiField; +import static org.elasticsearch.cluster.metadata.ProjectMetadataTests.assertLeafs; +import static org.elasticsearch.cluster.metadata.ProjectMetadataTests.assertMultiField; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; public class FieldFilterMapperPluginTests extends ESSingleNodeTestCase {