diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index da61d44264571..2c2aaaf322844 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -357,6 +357,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS = def(9_127_0_00); public static final TransportVersion ALLOCATION_DECISION_NOT_PREFERRED = def(9_145_0_00); public static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = def(9_146_0_00); + public static final TransportVersion PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY = def(9_147_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index 34d03bb37a92b..6c6723a25a201 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -91,7 +92,7 @@ protected void validateForReservedState(DeleteRepositoryRequest request, Cluster super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index a91a3c2217e88..35e44d9a67b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -95,7 +96,7 @@ protected void validateForReservedState(Request request, ClusterState state) { super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 79a8bffa1e1b6..ecd928a1bb5b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -130,7 +131,7 @@ protected void validateForReservedState(PutComponentTemplateAction.Request reque super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index a6fbd3db23e14..2ca82a3ddbd68 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -87,7 +88,10 @@ protected void masterOperation( ) { ProjectId projectId = projectResolver.getProjectId(); verifyIfUsingReservedComponentTemplates(request, state.metadata().reservedStateMetadata().values()); - verifyIfUsingReservedComponentTemplates(request, state.metadata().getProject(projectId).reservedStateMetadata().values()); + verifyIfUsingReservedComponentTemplates( + request, + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values() + ); ComposableIndexTemplate indexTemplate = request.indexTemplate(); indexTemplateService.putIndexTemplateV2( request.cause(), @@ -138,7 +142,7 @@ protected void validateForReservedState(Request request, ClusterState state) { super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java index 5ffef099833a2..5f2fac1e079c4 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.injection.guice.Inject; @@ -80,7 +81,7 @@ protected void validateForReservedState(PutPipelineRequest request, ClusterState super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index d986e4c285ed7..a8372cd07be25 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -61,7 +61,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.BiPredicate; import java.util.function.Consumer; @@ -799,12 +798,6 @@ private Iterator toXContentChunkedWithSingleProjectFormat( @FixForMultiProject final ProjectMetadata project = projectMetadata.values().iterator().next(); - // need to combine reserved state together into a single block so we don't get duplicate keys - // and not include it in the project xcontent output (through the lack of multi-project params) - // use a tree map so the order is deterministic - final Map clusterReservedState = new TreeMap<>(reservedStateMetadata); - clusterReservedState.putAll(project.reservedStateMetadata()); - // Similarly, combine cluster and project persistent tasks and report them under a single key Iterator customs = Iterators.flatMap(customs().entrySet().iterator(), entry -> { if (entry.getValue().context().contains(context) && ClusterPersistentTasksCustomMetadata.TYPE.equals(entry.getKey()) == false) { @@ -824,13 +817,20 @@ private Iterator toXContentChunkedWithSingleProjectFormat( ); } + // make order deterministic + Iterator reservedStateMetadataIterator = reservedStateMetadata.entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue) + .iterator(); + return Iterators.concat( start, clusterCoordination, persistentSettings, project.toXContentChunked(p), customs, - ChunkedToXContentHelper.object("reserved_state", clusterReservedState.values().iterator()), + ChunkedToXContentHelper.object("reserved_state", reservedStateMetadataIterator), ChunkedToXContentHelper.endObject() ); } @@ -845,6 +845,7 @@ private static class MetadataDiff implements Diff { private final Settings persistentSettings; private final Diff hashesOfConsistentSettings; private final ProjectMetadata.ProjectMetadataDiff singleProject; + private final MapDiff> multiProject; private final MapDiff> clusterCustoms; private final MapDiff> reservedStateMetadata; @@ -981,7 +982,7 @@ private MetadataDiff(StreamInput in) throws IOException { RESERVED_DIFF_VALUE_READER ); - singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms, DiffableUtils.emptyDiff()); + singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms); multiProject = null; } else { fromNodeBeforeMultiProjectsSupport = false; @@ -1048,7 +1049,7 @@ public void writeTo(StreamOutput out) throws IOException { singleProject.indices().writeTo(out); singleProject.templates().writeTo(out); buildUnifiedCustomDiff().writeTo(out); - buildUnifiedReservedStateMetadataDiff().writeTo(out); + reservedStateMetadata.writeTo(out); } else { clusterCustoms.writeTo(out); reservedStateMetadata.writeTo(out); @@ -1094,15 +1095,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - private Diff> buildUnifiedReservedStateMetadataDiff() { - return DiffableUtils.merge( - reservedStateMetadata, - singleProject.reservedStateMetadata(), - DiffableUtils.getStringKeySerializer(), - RESERVED_DIFF_VALUE_READER - ); - } - @Override public Metadata apply(Metadata part) { if (empty) { @@ -1304,12 +1296,7 @@ public void writeTo(StreamOutput out) throws IOException { ); VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms); - List combinedMetadata = new ArrayList<>( - reservedStateMetadata.size() + singleProject.reservedStateMetadata().size() - ); - combinedMetadata.addAll(reservedStateMetadata.values()); - combinedMetadata.addAll(singleProject.reservedStateMetadata().values()); - out.writeCollection(combinedMetadata); + out.writeCollection(reservedStateMetadata.values()); } else { VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 4b574b8313c28..a5995dd15490c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -78,6 +79,7 @@ import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.elasticsearch.cluster.metadata.Metadata.ALL; +import static org.elasticsearch.cluster.project.ProjectStateRegistry.RESERVED_DIFF_VALUE_READER; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; public class ProjectMetadata implements Iterable, Diffable, ChunkedToXContent { @@ -91,7 +93,6 @@ public class ProjectMetadata implements Iterable, Diffable> aliasedIndices; private final ImmutableOpenMap templates; private final ImmutableOpenMap customs; - private final ImmutableOpenMap reservedStateMetadata; private final int totalNumberOfShards; private final int totalOpenIndexShards; @@ -125,7 +126,6 @@ private ProjectMetadata( ImmutableOpenMap> aliasedIndices, ImmutableOpenMap templates, ImmutableOpenMap customs, - ImmutableOpenMap reservedStateMetadata, int totalNumberOfShards, int totalOpenIndexShards, String[] allIndices, @@ -143,7 +143,6 @@ private ProjectMetadata( this.aliasedIndices = aliasedIndices; this.templates = templates; this.customs = customs; - this.reservedStateMetadata = reservedStateMetadata; this.totalNumberOfShards = totalNumberOfShards; this.totalOpenIndexShards = totalOpenIndexShards; this.allIndices = allIndices; @@ -224,7 +223,6 @@ public ProjectMetadata withLifecycleState(Index index, LifecycleExecutionState l aliasedIndices, templates, customs, - reservedStateMetadata, totalNumberOfShards, totalOpenIndexShards, allIndices, @@ -257,7 +255,6 @@ public ProjectMetadata withIndexSettingsUpdates(Map updates) { aliasedIndices, templates, customs, - reservedStateMetadata, totalNumberOfShards, totalOpenIndexShards, allIndices, @@ -291,7 +288,6 @@ public ProjectMetadata withAllocationAndTermUpdatesOnly(Map customs() { return customs; } - public Map reservedStateMetadata() { - return reservedStateMetadata; - } - public Map dataStreams() { return custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY).dataStreams(); } @@ -1143,7 +1134,6 @@ public static class Builder { private final ImmutableOpenMap.Builder indices; private final ImmutableOpenMap.Builder templates; private final ImmutableOpenMap.Builder customs; - private final ImmutableOpenMap.Builder reservedStateMetadata; private SortedMap previousIndicesLookup; @@ -1160,7 +1150,6 @@ public static class Builder { this.indices = ImmutableOpenMap.builder(projectMetadata.indices); this.templates = ImmutableOpenMap.builder(projectMetadata.templates); this.customs = ImmutableOpenMap.builder(projectMetadata.customs); - this.reservedStateMetadata = ImmutableOpenMap.builder(projectMetadata.reservedStateMetadata); this.previousIndicesLookup = projectMetadata.indicesLookup; this.mappingsByHash = new HashMap<>(projectMetadata.mappingsByHash); this.checkForUnusedMappings = false; @@ -1174,7 +1163,6 @@ public static class Builder { indices = ImmutableOpenMap.builder(indexCountHint); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); - reservedStateMetadata = ImmutableOpenMap.builder(); previousIndicesLookup = null; this.mappingsByHash = new HashMap<>(mappingsByHash); indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize @@ -1519,21 +1507,6 @@ public Builder customs(Map customs) { return this; } - public Builder put(Map reservedStateMetadata) { - this.reservedStateMetadata.putAllFromMap(reservedStateMetadata); - return this; - } - - public Builder put(ReservedStateMetadata metadata) { - reservedStateMetadata.put(metadata.namespace(), metadata); - return this; - } - - public Builder removeReservedState(ReservedStateMetadata metadata) { - reservedStateMetadata.remove(metadata.namespace()); - return this; - } - public Builder indexGraveyard(final IndexGraveyard indexGraveyard) { return putCustom(IndexGraveyard.TYPE, indexGraveyard); } @@ -1682,7 +1655,6 @@ public ProjectMetadata build(boolean skipNameCollisionChecks) { aliasedIndices, templates.build(), customs.build(), - reservedStateMetadata.build(), totalNumberOfShards, totalOpenIndexShards, allIndicesArray, @@ -2087,6 +2059,7 @@ static boolean assertDataStreams(Map indices, DataStreamM return true; } + @FixForMultiProject(description = "Remove reading reserved_state and settings") // ES-12795 public static ProjectMetadata fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.currentToken(); if (token == null) { @@ -2107,9 +2080,10 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep } } else if (token == XContentParser.Token.START_OBJECT) { switch (currentFieldName) { + // Remove this (ES-12795) case "reserved_state" -> { while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - projectBuilder.put(ReservedStateMetadata.fromXContent(parser)); + ReservedStateMetadata.fromXContent(parser); } } case "indices" -> { @@ -2122,6 +2096,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep projectBuilder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); } } + // Remove this (ES-12795) case "settings" -> { Settings.fromXContent(parser); } @@ -2169,10 +2144,7 @@ public Iterator toXContentChunked(ToXContent.Params p) { ) ), indices, - customs, - multiProject - ? ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator()) - : Collections.emptyIterator() + customs ); } @@ -2199,9 +2171,11 @@ public static ProjectMetadata readFrom(StreamInput in) throws IOException { readProjectCustoms(in, builder); - int reservedStateSize = in.readVInt(); - for (int i = 0; i < reservedStateSize; i++) { - builder.put(ReservedStateMetadata.readFrom(in)); + if (in.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + int reservedStateSize = in.readVInt(); + for (int i = 0; i < reservedStateSize; i++) { + ReservedStateMetadata.readFrom(in); + } } if (in.getTransportVersion() @@ -2238,7 +2212,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeCollection(templates.values()); VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); - out.writeCollection(reservedStateMetadata.values()); + if (out.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + out.writeCollection(Collections.emptySet()); + } if (out.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { @@ -2253,23 +2229,16 @@ static class ProjectMetadataDiff implements Diff { new DiffableUtils.DiffableValueReader<>(IndexMetadata::readFrom, IndexMetadata::readDiffFrom); private static final DiffableUtils.DiffableValueReader TEMPLATES_DIFF_VALUE_READER = new DiffableUtils.DiffableValueReader<>(IndexTemplateMetadata::readFrom, IndexTemplateMetadata::readDiffFrom); - private static final DiffableUtils.DiffableValueReader RESERVED_DIFF_VALUE_READER = - new DiffableUtils.DiffableValueReader<>(ReservedStateMetadata::readFrom, ReservedStateMetadata::readDiffFrom); private final DiffableUtils.MapDiff> indices; private final DiffableUtils.MapDiff> templates; private final DiffableUtils.MapDiff> customs; - private final DiffableUtils.MapDiff< - String, - ReservedStateMetadata, - ImmutableOpenMap> reservedStateMetadata; private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) { if (before == after) { indices = DiffableUtils.emptyDiff(); templates = DiffableUtils.emptyDiff(); customs = DiffableUtils.emptyDiff(); - reservedStateMetadata = DiffableUtils.emptyDiff(); } else { indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer()); @@ -2279,35 +2248,26 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) { DiffableUtils.getStringKeySerializer(), PROJECT_CUSTOM_VALUE_SERIALIZER ); - reservedStateMetadata = DiffableUtils.diff( - before.reservedStateMetadata, - after.reservedStateMetadata, - DiffableUtils.getStringKeySerializer() - ); } } ProjectMetadataDiff( DiffableUtils.MapDiff> indices, DiffableUtils.MapDiff> templates, - DiffableUtils.MapDiff> customs, - DiffableUtils.MapDiff> reservedStateMetadata + DiffableUtils.MapDiff> customs ) { this.indices = indices; this.templates = templates; this.customs = customs; - this.reservedStateMetadata = reservedStateMetadata; } ProjectMetadataDiff(StreamInput in) throws IOException { indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), INDEX_METADATA_DIFF_VALUE_READER); templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), TEMPLATES_DIFF_VALUE_READER); customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), PROJECT_CUSTOM_VALUE_SERIALIZER); - reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff( - in, - DiffableUtils.getStringKeySerializer(), - RESERVED_DIFF_VALUE_READER - ); + if (in.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), RESERVED_DIFF_VALUE_READER); + } if (in.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { Settings.readSettingsDiffFromStream(in); @@ -2326,16 +2286,14 @@ DiffableUtils.MapDiff> reservedStateMetadata() { - return reservedStateMetadata; - } - @Override public void writeTo(StreamOutput out) throws IOException { indices.writeTo(out); templates.writeTo(out); customs.writeTo(out); - reservedStateMetadata.writeTo(out); + if (out.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + DiffableUtils.emptyDiff().writeTo(out); + } if (out.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { Settings.EMPTY_DIFF.writeTo(out); @@ -2344,7 +2302,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ProjectMetadata apply(ProjectMetadata part) { - if (indices.isEmpty() && templates.isEmpty() && customs.isEmpty() && reservedStateMetadata.isEmpty()) { + if (indices.isEmpty() && templates.isEmpty() && customs.isEmpty()) { // nothing to do return part; } @@ -2354,7 +2312,6 @@ public ProjectMetadata apply(ProjectMetadata part) { builder.indices(updatedIndices); builder.templates(templates.apply(part.templates)); builder.customs(customs.apply(part.customs)); - builder.put(reservedStateMetadata.apply(part.reservedStateMetadata)); if (part.indices == updatedIndices && builder.dataStreamMetadata() == part.custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY)) { builder.previousIndicesLookup = part.indicesLookup; diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index 464c28a4cc834..dd85bd0e68c70 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.NamedDiffable; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -29,6 +30,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -38,6 +40,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; /** @@ -46,7 +49,10 @@ public class ProjectStateRegistry extends AbstractNamedDiffable implements Custom, NamedDiffable { public static final String TYPE = "projects_registry"; public static final ProjectStateRegistry EMPTY = new ProjectStateRegistry(Collections.emptyMap(), Collections.emptySet(), 0); - private static final Entry EMPTY_ENTRY = new Entry(Settings.EMPTY); + private static final Entry EMPTY_ENTRY = new Entry(Settings.EMPTY, ImmutableOpenMap.of()); + + public static final DiffableUtils.DiffableValueReader RESERVED_DIFF_VALUE_READER = + new DiffableUtils.DiffableValueReader<>(ReservedStateMetadata::readFrom, ReservedStateMetadata::readDiffFrom); private final Map projectsEntries; // Projects that have been marked for deletion based on their file-based setting @@ -63,7 +69,9 @@ public ProjectStateRegistry(StreamInput in) throws IOException { projectsEntries = in.readMap(ProjectId::readFrom, Entry::readFrom); } else { Map settingsMap = in.readMap(ProjectId::readFrom, Settings::readSettingsFromStream); - projectsEntries = settingsMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new Entry(e.getValue()))); + projectsEntries = settingsMap.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new Entry(e.getValue(), ImmutableOpenMap.of()))); } if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_STATE_REGISTRY_RECORDS_DELETIONS)) { projectsMarkedForDeletion = in.readCollectionAsImmutableSet(ProjectId::readFrom); @@ -105,6 +113,10 @@ public Settings getProjectSettings(ProjectId projectId) { return projectsEntries.getOrDefault(projectId, EMPTY_ENTRY).settings; } + public Map reservedStateMetadata(ProjectId projectId) { + return projectsEntries.getOrDefault(projectId, EMPTY_ENTRY).reservedStateMetadata; + } + public Set getProjectsMarkedForDeletion() { return projectsMarkedForDeletion; } @@ -304,14 +316,22 @@ private Builder(ProjectStateRegistry original) { this.projectsMarkedForDeletionGeneration = original.projectsMarkedForDeletionGeneration; } - public Builder putProjectSettings(ProjectId projectId, Settings settings) { + private void updateEntry(ProjectId projectId, UnaryOperator modifier) { Entry entry = projectsEntries.get(projectId); if (entry == null) { - entry = new Entry(settings); - } else { - entry = entry.withSettings(settings); + entry = new Entry(); } + entry = modifier.apply(entry); projectsEntries.put(projectId, entry); + } + + public Builder putProjectSettings(ProjectId projectId, Settings settings) { + updateEntry(projectId, entry -> entry.withSettings(settings)); + return this; + } + + public Builder putReservedStateMetadata(ProjectId projectId, ReservedStateMetadata reservedStateMetadata) { + updateEntry(projectId, entry -> entry.withReservedStateMetadata(reservedStateMetadata)); return this; } @@ -343,25 +363,66 @@ public ProjectStateRegistry build() { } } - private record Entry(Settings settings) implements Writeable, Diffable { + private record Entry(Settings settings, ImmutableOpenMap reservedStateMetadata) + implements + ToXContentFragment, + Writeable, + Diffable { + + Entry() { + this(Settings.EMPTY, ImmutableOpenMap.of()); + } public static Entry readFrom(StreamInput in) throws IOException { - return new Entry(Settings.readSettingsFromStream(in)); + Settings settings = Settings.readSettingsFromStream(in); + + ImmutableOpenMap reservedStateMetadata; + if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + int reservedStateSize = in.readVInt(); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(reservedStateSize); + for (int i = 0; i < reservedStateSize; i++) { + ReservedStateMetadata r = ReservedStateMetadata.readFrom(in); + builder.put(r.namespace(), r); + } + reservedStateMetadata = builder.build(); + } else { + reservedStateMetadata = ImmutableOpenMap.of(); + } + + return new Entry(settings, reservedStateMetadata); } public Entry withSettings(Settings settings) { - return new Entry(settings); + return new Entry(settings, reservedStateMetadata); + } + + public Entry withReservedStateMetadata(ReservedStateMetadata reservedStateMetadata) { + ImmutableOpenMap reservedStateMetadataMap = ImmutableOpenMap.builder(this.reservedStateMetadata) + .fPut(reservedStateMetadata.namespace(), reservedStateMetadata) + .build(); + return new Entry(settings, reservedStateMetadataMap); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeWriteable(settings); + if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + out.writeCollection(reservedStateMetadata.values()); + } } - public void toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject("settings"); settings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); builder.endObject(); + + builder.startObject("reserved_state"); + for (ReservedStateMetadata reservedStateMetadata : reservedStateMetadata.values()) { + reservedStateMetadata.toXContent(builder, params); + } + builder.endObject(); + return builder; } @Override @@ -369,22 +430,46 @@ public Diff diff(Entry previousState) { if (this == previousState) { return SimpleDiffable.empty(); } - return new EntryDiff(settings.diff(previousState.settings)); + + return new EntryDiff( + settings.diff(previousState.settings), + DiffableUtils.diff(previousState.reservedStateMetadata, reservedStateMetadata, DiffableUtils.getStringKeySerializer()) + ); } - private record EntryDiff(Diff settingsDiff) implements Diff { + private record EntryDiff( + Diff settingsDiff, + DiffableUtils.MapDiff> reservedStateMetadata + ) implements Diff { + public static EntryDiff readFrom(StreamInput in) throws IOException { - return new EntryDiff(Settings.readSettingsDiffFromStream(in)); + Diff settingsDiff = Settings.readSettingsDiffFromStream(in); + + DiffableUtils.MapDiff> reservedStateMetadata; + if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + RESERVED_DIFF_VALUE_READER + ); + } else { + reservedStateMetadata = DiffableUtils.emptyDiff(); + } + + return new EntryDiff(settingsDiff, reservedStateMetadata); } @Override public Entry apply(Entry part) { - return part.withSettings(settingsDiff.apply(part.settings)); + return new Entry(settingsDiff.apply(part.settings), reservedStateMetadata.apply(part.reservedStateMetadata)); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeWriteable(settingsDiff); + if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + reservedStateMetadata.writeTo(out); + } } } } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index f6f2131f79d3d..47182c30dd30b 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.env.Environment; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorImpact; @@ -125,11 +126,19 @@ public Path watchedFile() { *

* If there's no file based settings file in this cluster, we'll remove all state reservations for * file based settings from the cluster state. + * * @param clusterState the cluster state before snapshot restore - * @param mdBuilder the current metadata builder for the new cluster state - * @param projectId the project associated with the restore + * @param builder the current ClusterState builder for the new cluster state + * @param mdBuilder the current metadata builder for the new cluster state + * @param projectId the project associated with the restore */ - public void handleSnapshotRestore(ClusterState clusterState, Metadata.Builder mdBuilder, ProjectId projectId) { + @FixForMultiProject(description = "Simplify parameters (ES-12796)") + public void handleSnapshotRestore( + ClusterState clusterState, + ClusterState.Builder builder, + Metadata.Builder mdBuilder, + ProjectId projectId + ) { assert clusterState.nodes().isLocalNodeElectedMaster(); ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(NAMESPACE); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java index e106d3f44a030..180e0365e7e55 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; @@ -451,7 +452,7 @@ public void process( ProjectMetadata projectMetadata = getPotentiallyNewProject(state, projectId); state = ClusterState.builder(state).putProjectMetadata(projectMetadata).build(); - ReservedStateMetadata existingMetadata = projectMetadata.reservedStateMetadata().get(namespace); + ReservedStateMetadata existingMetadata = ProjectStateRegistry.get(state).reservedStateMetadata(projectId).get(namespace); // We check if we should exit early on the state version from clusterService. The ReservedStateUpdateTask // will check again with the most current state version if this continues. diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java index db564ed2a838c..1b3e2661480f4 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java @@ -65,7 +65,7 @@ protected ClusterState execute(ClusterState currentState) { ProjectMetadata currentProject = ReservedClusterStateService.getPotentiallyNewProject(currentState, projectId); var result = execute( ClusterState.builder(currentState).putProjectMetadata(currentProject).build(), - currentProject.reservedStateMetadata() + ProjectStateRegistry.get(currentState).reservedStateMetadata(projectId) ); if (result == null) { return currentState; @@ -78,8 +78,11 @@ protected ClusterState execute(ClusterState currentState) { ); ProjectMetadata updatedProjectMetadata = updatedClusterState.getMetadata().getProject(projectId); return ClusterState.builder(currentState) - .putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(updatedProjectStateRegistry).build()) - .putProjectMetadata(ProjectMetadata.builder(updatedProjectMetadata).put(result.v2())) + .putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder(updatedProjectStateRegistry).putReservedStateMetadata(projectId, result.v2()).build() + ) + .putProjectMetadata(updatedProjectMetadata) .build(); } } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java index 225e491013606..3113505561e46 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java @@ -16,9 +16,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import static org.elasticsearch.cluster.metadata.ReservedStateMetadata.EMPTY_VERSION; import static org.elasticsearch.cluster.metadata.ReservedStateMetadata.NO_VERSION; @@ -63,7 +64,7 @@ static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateV static ReservedStateMetadata getMetadata(ClusterState state, ErrorState errorState) { return errorState.projectId() - .map(p -> ReservedClusterStateService.getPotentiallyNewProject(state, p).reservedStateMetadata()) + .map(ProjectStateRegistry.get(state)::reservedStateMetadata) .orElseGet(() -> state.metadata().reservedStateMetadata()) .get(errorState.namespace()); } @@ -93,13 +94,17 @@ ClusterState execute(ClusterState currentState) { var errorMetadata = new ReservedStateErrorMetadata(errorState.version(), errorState.errorKind(), errorState.errors()); if (errorState.projectId().isPresent()) { - ProjectMetadata project = currentState.metadata().getProject(errorState.projectId().get()); + ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(currentState); - ReservedStateMetadata reservedMetadata = project.reservedStateMetadata().get(errorState.namespace()); + ProjectId projectId = errorState.projectId().get(); + ReservedStateMetadata reservedMetadata = projectStateRegistry.reservedStateMetadata(projectId).get(errorState.namespace()); ReservedStateMetadata.Builder resBuilder = ReservedStateMetadata.builder(errorState.namespace(), reservedMetadata); resBuilder.errorMetadata(errorMetadata); - stateBuilder.putProjectMetadata(ProjectMetadata.builder(project).put(resBuilder.build())); + stateBuilder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder(projectStateRegistry).putReservedStateMetadata(projectId, resBuilder.build()).build() + ); } else { Metadata.Builder metadataBuilder = Metadata.builder(currentState.metadata()); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index d65af4bc9b64c..1be7411bcca97 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1573,7 +1573,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) { // Restore global state if needed if (request.includeGlobalState()) { applyGlobalStateRestore(currentState, mdBuilder, projectId); - fileSettingsService.handleSnapshotRestore(currentState, mdBuilder, projectId); + fileSettingsService.handleSnapshotRestore(currentState, builder, mdBuilder, projectId); } if (completed(shards)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index cb9fa23aaefbc..8513cf5eff45f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -754,8 +755,9 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { var updatedState = processJSON(action, prevState, settingsJSON); - ProjectMetadata withReservedState = ProjectMetadata.builder(updatedState.state().getMetadata().getProject(projectId)) - .put( + ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()) + .putReservedStateMetadata( + projectId, ReservedStateMetadata.builder("test") .putHandler(new ReservedStateHandlerMetadata(ReservedComposableIndexTemplateAction.NAME, updatedState.keys())) .build() @@ -810,7 +812,7 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { IllegalArgumentException.class, () -> TransportPutComposableIndexTemplateAction.verifyIfUsingReservedComponentTemplates( request, - withReservedState.reservedStateMetadata().values() + withReservedState.reservedStateMetadata(projectId).values() ) ).getMessage().contains("errors: [[component_template:template_1] is reserved by [test]]") ); @@ -824,7 +826,7 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { // this should just work, no failure TransportPutComposableIndexTemplateAction.verifyIfUsingReservedComponentTemplates( request, - withReservedState.reservedStateMetadata().values() + withReservedState.reservedStateMetadata(projectId).values() ); } } @@ -922,8 +924,9 @@ public void testTemplatesWithReservedPrefix() throws Exception { allOf(aMapWithSize(2), hasKey(reservedComposableIndexName(conflictingTemplateName)), hasKey(conflictingTemplateName)) ); - ProjectMetadata withReservedMetadata = ProjectMetadata.builder(updatedState.state().getMetadata().getProject(projectId)) - .put( + ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()) + .putReservedStateMetadata( + projectId, new ReservedStateMetadata.Builder("file_settings").putHandler( new ReservedStateHandlerMetadata(ReservedComposableIndexTemplateAction.NAME, updatedState.keys()) ).build() @@ -957,7 +960,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { expectThrows( IllegalArgumentException.class, () -> fakeAction.validateForReservedState( - withReservedMetadata.reservedStateMetadata().values(), + withReservedState.reservedStateMetadata(projectId).values(), ReservedComposableIndexTemplateAction.NAME, modifiedKeys, pr.name() @@ -973,7 +976,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { assertThat(modifiedKeysOK, hasSize(1)); fakeAction.validateForReservedState( - withReservedMetadata.reservedStateMetadata().values(), + withReservedState.reservedStateMetadata(projectId).values(), ReservedComposableIndexTemplateAction.NAME, modifiedKeysOK, prOK.name() diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 78623dd3f2738..3d8461bc4d29d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -26,6 +26,8 @@ import org.elasticsearch.cluster.metadata.MetadataTests; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -44,6 +46,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -85,8 +90,10 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -515,8 +522,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "event_ingested_range": { "shards": [] } } }, - "index-graveyard": { "tombstones": [] }, - "reserved_state": {} + "index-graveyard": { "tombstones": [] } }, { "id": "3LftaL7hgfXAsF60Gm6jcD", @@ -573,15 +579,13 @@ public void testToXContentWithMultipleProjects() throws IOException { "event_ingested_range": { "shards": [] } } }, - "index-graveyard": { "tombstones": [] }, - "reserved_state": {} + "index-graveyard": { "tombstones": [] } }, { "id": "WHyuJ0uqBYOPgHX9kYUXlZ", "templates": {}, "indices": {}, - "index-graveyard": { "tombstones": [] }, - "reserved_state": {} + "index-graveyard": { "tombstones": [] } } ], "reserved_state": {} @@ -810,6 +814,17 @@ public void testToXContentWithMultipleProjects() throws IOException { "project.setting": "42", "project.setting2": "43" }, + "reserved_state": { + "file_settings": { + "handlers": { + "settings": { + "keys": ["project.setting", "project.setting2"] + } + }, + "version": 42, + "errors": null + } + }, "marked_for_deletion": true } ], @@ -929,6 +944,15 @@ private static ClusterState buildMultiProjectClusterState(DiscoveryNode... nodes projectId1, Settings.builder().put(PROJECT_SETTING.getKey(), 42).put(PROJECT_SETTING2.getKey(), 43).build() ) + .putReservedStateMetadata( + projectId1, + ReservedStateMetadata.builder("file_settings") + .putHandler( + new ReservedStateHandlerMetadata("settings", Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey())) + ) + .version(42L) + .build() + ) .markProjectForDeletion(projectId1) .build() ) @@ -2241,4 +2265,25 @@ public static int expectedChunkCount(ToXContent.Params params, ClusterState clus return Math.toIntExact(chunkCount); } + + public void testSerialization() throws IOException { + ClusterState clusterState = buildClusterState(); + BytesStreamOutput out = new BytesStreamOutput(); + clusterState.writeTo(out); + + // check it deserializes ok + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + ClusterState deserialisedClusterState = ClusterState.readFrom( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry), + null + ); + + // check it matches the original object + Metadata deserializedMetadata = deserialisedClusterState.metadata(); + assertThat(deserializedMetadata.projects(), aMapWithSize(1)); + assertThat(deserializedMetadata.projects(), hasKey(ProjectId.DEFAULT)); + + assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).templates(), hasKey("template")); + assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).indices(), hasKey("index")); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 373831b54804b..3419b30bd2e89 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -969,10 +969,6 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata .sum(); int reservedStateSize = metadata.reservedStateMetadata().size(); - if (params.paramAsBoolean("multi-project", false) == false) { - // only one project if not multi-project, add its reserved state to the cluster's collection - reservedStateSize += metadata.projects().values().iterator().next().reservedStateMetadata().size(); - } // 2 chunks for wrapping reserved state + 1 chunk for each item chunkCount += 2 + reservedStateSize; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index b92c9ada0bc28..366faa8817f60 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -2745,8 +2745,7 @@ public void testToXContentMultiProject() throws IOException { } }, "data_stream_aliases": {} - }, - "reserved_state": {} + } } """, IndexVersion.current(), @@ -2849,11 +2848,6 @@ static int expectedChunkCount(ToXContent.Params params, ProjectMetadata project) } } - if (params.paramAsBoolean("multi-project", false)) { - // 2 chunks for wrapping reserved state + 1 chunk for each item - chunkCount += 2 + project.reservedStateMetadata().size(); - } - return Math.toIntExact(chunkCount); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index e10012133ffcd..b133bbde7552a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -127,13 +127,11 @@ public void testSimpleJsonFromAndTo() throws IOException { .put(idx2, false) .put(DataStreamTestHelper.newInstance("data-stream1", List.of(idx1.getIndex()))) .put(DataStreamTestHelper.newInstance("data-stream2", List.of(idx2.getIndex()))) - .put(reservedStateMetadata) - .put(reservedStateMetadata1) .build(); XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); - ChunkedToXContent.wrapAsToXContent(Metadata.builder().put(project).build()) + ChunkedToXContent.wrapAsToXContent(Metadata.builder().put(project).put(reservedStateMetadata).put(reservedStateMetadata1).build()) .toXContent( builder, new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY)) @@ -282,8 +280,7 @@ public void testToXContentGateway_MultiProject() throws IOException { }, "index-graveyard" : { "tombstones" : [ ] - }, - "reserved_state" : { } + } } ], "reserved_state" : { } diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index 7e8e262f954dc..987da3c95805d 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -453,7 +453,7 @@ public void testHandleSnapshotRestoreClearsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(state.metadata()); - fileSettingsService.handleSnapshotRestore(state, metadata, ProjectId.DEFAULT); + fileSettingsService.handleSnapshotRestore(state, ClusterState.builder(state), metadata, ProjectId.DEFAULT); assertThat(metadata.build().reservedStateMetadata(), anEmptyMap()); } @@ -476,7 +476,7 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(); - fileSettingsService.handleSnapshotRestore(state, metadata, ProjectId.DEFAULT); + fileSettingsService.handleSnapshotRestore(state, ClusterState.builder(state), metadata, ProjectId.DEFAULT); assertThat( metadata.build().reservedStateMetadata(), diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java index 981dd5e9475a2..ac38d3e1935be 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; @@ -183,7 +184,7 @@ private static ClusterState setupProject(ClusterState state, Optional } private static Map getMetadata(ClusterState state, Optional projectId) { - return projectId.map(p -> state.metadata().getProject(p).reservedStateMetadata()) + return projectId.map(p -> ProjectStateRegistry.get(state).reservedStateMetadata(p)) .orElseGet(() -> state.metadata().reservedStateMetadata()); } @@ -739,10 +740,16 @@ public void testUpdateTaskDuplicateError() { Optional projectId = randomBoolean() ? Optional.empty() : Optional.of(randomProjectIdOrDefault()); - Metadata metadata = projectId.map(p -> Metadata.builder().put(ProjectMetadata.builder(p).put(operatorMetadata))) - .orElseGet(() -> Metadata.builder().put(operatorMetadata)) - .build(); - ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); + ClusterState.Builder builder = ClusterState.builder(new ClusterName("test")); + if (projectId.isPresent()) { + builder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build() + ); + } else { + builder.metadata(Metadata.builder().put(operatorMetadata)); + } + ClusterState state = builder.build(); assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 2L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 1L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); @@ -849,10 +856,16 @@ public TransformState transform(Map source, TransformState prevS .putHandler(hmOne) .build(); - metadata = projectId.map(p -> Metadata.builder().put(ProjectMetadata.builder(p).put(opMetadata))) - .orElseGet(() -> Metadata.builder().put(opMetadata)) - .build(); - ClusterState newState = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); + builder = ClusterState.builder(new ClusterName("test")); + if (projectId.isPresent()) { + builder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), opMetadata).build() + ); + } else { + builder.metadata(Metadata.builder().put(opMetadata)); + } + ClusterState newState = builder.build(); // We exit on duplicate errors before we update the reserved state error metadata assertThat( @@ -865,10 +878,16 @@ public void testCheckMetadataVersion() { ReservedStateMetadata operatorMetadata = ReservedStateMetadata.builder("test").version(123L).build(); Optional projectId = randomBoolean() ? Optional.empty() : Optional.of(randomProjectIdOrDefault()); - Metadata metadata = projectId.map(p -> Metadata.builder().put(ProjectMetadata.builder(p).put(operatorMetadata))) - .orElseGet(() -> Metadata.builder().put(operatorMetadata)) - .build(); - ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); + ClusterState.Builder builder = ClusterState.builder(new ClusterName("test")); + if (projectId.isPresent()) { + builder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build() + ); + } else { + builder.metadata(Metadata.builder().put(operatorMetadata)); + } + ClusterState state = builder.build(); ReservedStateUpdateTask task = createEmptyTask( projectId,