From df3d4d6f2eec89899822d68b96cb6fee3078d151 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 1 Aug 2025 14:44:50 +0100 Subject: [PATCH 01/16] Projects reserved state is moved to ProjectStateRegistry --- .../org/elasticsearch/TransportVersions.java | 1 + .../TransportDeleteRepositoryAction.java | 3 +- ...rtDeleteComposableIndexTemplateAction.java | 3 +- .../TransportPutComponentTemplateAction.java | 3 +- ...sportPutComposableIndexTemplateAction.java | 5 +- .../ingest/PutPipelineTransportAction.java | 3 +- .../elasticsearch/cluster/ClusterState.java | 35 +++++- .../elasticsearch/cluster/DiffableUtils.java | 10 +- .../cluster/metadata/Metadata.java | 40 ++++--- .../cluster/metadata/ProjectMetadata.java | 89 ++++----------- .../cluster/project/ProjectStateRegistry.java | 105 +++++++++++++++--- .../service/FileSettingsService.java | 8 +- .../service/ReservedClusterStateService.java | 3 +- .../ReservedProjectStateUpdateTask.java | 6 +- .../service/ReservedStateErrorTask.java | 14 ++- .../snapshots/RestoreService.java | 2 +- ...vedComposableIndexTemplateActionTests.java | 17 +-- .../cluster/metadata/MetadataTests.java | 4 - .../metadata/ProjectMetadataTests.java | 5 - .../metadata/ToAndFromJsonMetadataTests.java | 2 - .../service/FileSettingsServiceTests.java | 4 +- .../ReservedClusterStateServiceTests.java | 37 +++--- 22 files changed, 246 insertions(+), 153 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index dad300ae72744..a5d9747263ae1 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -352,6 +352,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_TOPN_TIMINGS = def(9_128_0_00); public static final TransportVersion NODE_WEIGHTS_ADDED_TO_NODE_BALANCE_STATS = def(9_129_0_00); public static final TransportVersion RERANK_SNIPPETS = def(9_130_0_00); + public static final TransportVersion PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY = def(9_131_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index 34d03bb37a92b..6c6723a25a201 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -91,7 +92,7 @@ protected void validateForReservedState(DeleteRepositoryRequest request, Cluster super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index a91a3c2217e88..35e44d9a67b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -95,7 +96,7 @@ protected void validateForReservedState(Request request, ClusterState state) { super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 05d055dbf979c..99c30dc2f66db 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -128,7 +129,7 @@ protected void validateForReservedState(PutComponentTemplateAction.Request reque super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index a6fbd3db23e14..9c18078ada71b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -87,7 +88,7 @@ protected void masterOperation( ) { ProjectId projectId = projectResolver.getProjectId(); verifyIfUsingReservedComponentTemplates(request, state.metadata().reservedStateMetadata().values()); - verifyIfUsingReservedComponentTemplates(request, state.metadata().getProject(projectId).reservedStateMetadata().values()); + verifyIfUsingReservedComponentTemplates(request, ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values()); ComposableIndexTemplate indexTemplate = request.indexTemplate(); indexTemplateService.putIndexTemplateV2( request.cause(), @@ -138,7 +139,7 @@ protected void validateForReservedState(Request request, ClusterState state) { super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java index e6c1b00ccb815..d90855ff05bf1 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.injection.guice.Inject; @@ -96,7 +97,7 @@ protected void validateForReservedState(PutPipelineRequest request, ClusterState super.validateForReservedState(request, state); validateForReservedState( - projectResolver.getProjectMetadata(state).reservedStateMetadata().values(), + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values(), reservedStateHandlerName().get(), modifiedKeys(request), request.toString() diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index d5356bd54b845..c1d3dbbae83a5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -24,8 +24,10 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.GlobalRoutingTable; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingNodes; @@ -73,6 +75,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static org.elasticsearch.cluster.metadata.Metadata.DEFAULT_PROJECT_ID; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; /** @@ -1316,7 +1319,12 @@ public void writeTo(StreamOutput out) throws IOException { clusterName.writeTo(out); out.writeLong(version); out.writeString(stateUUID); - metadata.writeTo(out); + if (out.getTransportVersion().before(TransportVersions.MULTI_PROJECT)) { + Map singleProjectReservedState = ProjectStateRegistry.get(this).reservedStateMetadata(Metadata.DEFAULT_PROJECT_ID); + metadata.writeTo(out, singleProjectReservedState); + } else { + metadata.writeTo(out); + } if (out.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT)) { routingTable.writeTo(out); } else { @@ -1365,11 +1373,34 @@ private static class ClusterStateDiff implements Diff { COMPATIBILITY_VERSIONS_VALUE_SERIALIZER ); features = after.clusterFeatures.diff(before.clusterFeatures); - metadata = after.metadata.diff(before.metadata); + + metadata = getMetadataDiff(before, after); blocks = after.blocks.diff(before.blocks); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); } + @FixForMultiProject + private Diff getMetadataDiff(ClusterState before, ClusterState after) { + final Diff metadata; + ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(after); + if (projectStateRegistry.size() == 1 && projectStateRegistry.hasProject(Metadata.DEFAULT_PROJECT_ID)) { + Map reservedStateMetadataBefore = ProjectStateRegistry.get(before).reservedStateMetadata(DEFAULT_PROJECT_ID); + Map reservedStateMetadataAfter = projectStateRegistry.reservedStateMetadata(DEFAULT_PROJECT_ID); + DiffableUtils.MapDiff< + String, + ReservedStateMetadata, + Map> diff = DiffableUtils.diff( + reservedStateMetadataBefore, + reservedStateMetadataAfter, + DiffableUtils.getStringKeySerializer() + ); + metadata = after.metadata.diff(before.metadata, diff); + } else { + metadata = after.metadata.diff(before.metadata); + } + return metadata; + } + ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { clusterName = new ClusterName(in); fromUuid = in.readString(); diff --git a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java index 542f6879ffd41..aff5de0b1737d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java @@ -104,8 +104,8 @@ public static > MapDiff emptyDiff() { */ @SuppressWarnings("unchecked") public static , T1 extends T, T2 extends T, M extends Map> MapDiff merge( - MapDiff> diff1, - MapDiff> diff2, + MapDiff> diff1, + MapDiff> diff2, KeySerializer keySerializer ) { return merge(diff1, diff2, keySerializer, DiffableValueSerializer.getWriteOnlyInstance()); @@ -116,8 +116,8 @@ public static , T1 extends T, T2 extends T, M extends M */ @SuppressWarnings("unchecked") public static > MapDiff merge( - MapDiff> diff1, - MapDiff> diff2, + MapDiff> diff1, + MapDiff> diff2, KeySerializer keySerializer, ValueSerializer valueSerializer ) { @@ -130,7 +130,7 @@ public static > MapDiff (T) val), mapEntries(diff2.getUpserts(), val -> (T) val) ).toList(); - return new MapDiff(keySerializer, valueSerializer, deletes, diffs, upserts, DiffableUtils::createImmutableMapBuilder); + return new MapDiff<>(keySerializer, valueSerializer, deletes, diffs, upserts, DiffableUtils::createImmutableMapBuilder); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index d4bc58c299435..d82e0fbb7a406 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -61,7 +61,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; import java.util.function.BiConsumer; import java.util.function.BiPredicate; import java.util.function.Consumer; @@ -707,7 +706,12 @@ private static boolean projectMetadataEqual( @Override public Diff diff(Metadata previousState) { - return new MetadataDiff(previousState, this); + return diff(previousState, DiffableUtils.emptyDiff()); + } + + public Diff diff(Metadata previousState, + MapDiff> singleProjectReservedStateMetadata) { + return new MetadataDiff(previousState, this, singleProjectReservedStateMetadata); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -799,12 +803,6 @@ private Iterator toXContentChunkedWithSingleProjectFormat( @FixForMultiProject final ProjectMetadata project = projectMetadata.values().iterator().next(); - // need to combine reserved state together into a single block so we don't get duplicate keys - // and not include it in the project xcontent output (through the lack of multi-project params) - // use a tree map so the order is deterministic - final Map clusterReservedState = new TreeMap<>(reservedStateMetadata); - clusterReservedState.putAll(project.reservedStateMetadata()); - // Similarly, combine cluster and project persistent tasks and report them under a single key Iterator customs = Iterators.flatMap(customs().entrySet().iterator(), entry -> { if (entry.getValue().context().contains(context) && ClusterPersistentTasksCustomMetadata.TYPE.equals(entry.getKey()) == false) { @@ -830,7 +828,7 @@ private Iterator toXContentChunkedWithSingleProjectFormat( persistentSettings, project.toXContentChunked(p), customs, - ChunkedToXContentHelper.object("reserved_state", clusterReservedState.values().iterator()), + ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata.values().iterator()), ChunkedToXContentHelper.endObject() ); } @@ -845,6 +843,11 @@ private static class MetadataDiff implements Diff { private final Settings persistentSettings; private final Diff hashesOfConsistentSettings; private final ProjectMetadata.ProjectMetadataDiff singleProject; + private final DiffableUtils.MapDiff< + String, + ReservedStateMetadata, + Map> singleProjectReservedStateMetadata; + private final MapDiff> multiProject; private final MapDiff> clusterCustoms; private final MapDiff> reservedStateMetadata; @@ -861,7 +864,8 @@ private static class MetadataDiff implements Diff { // This is used only when the node has a single project and needs to send the diff to an old node (wire BWC). private final MapDiff> combinedTasksDiff; - MetadataDiff(Metadata before, Metadata after) { + MetadataDiff(Metadata before, Metadata after, + MapDiff> singleProjectReservedState) { this.empty = before == after; this.fromNodeBeforeMultiProjectsSupport = false; // diff on this node, always after multi-projects, even when disabled clusterUUID = after.clusterUUID; @@ -873,9 +877,11 @@ private static class MetadataDiff implements Diff { if (before.isSingleProject() && after.isSingleProject()) { // single-project, just handle the project metadata diff itself singleProject = after.getSingleProject().diff(before.getSingleProject()); + singleProjectReservedStateMetadata = singleProjectReservedState; multiProject = null; } else { singleProject = null; + singleProjectReservedStateMetadata = null; multiProject = DiffableUtils.diff(before.projectMetadata, after.projectMetadata, ProjectId.PROJECT_ID_SERIALIZER); } @@ -981,7 +987,8 @@ private MetadataDiff(StreamInput in) throws IOException { RESERVED_DIFF_VALUE_READER ); - singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms, DiffableUtils.emptyDiff()); + singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms); + singleProjectReservedStateMetadata = DiffableUtils.emptyDiff(); multiProject = null; } else { fromNodeBeforeMultiProjectsSupport = false; @@ -998,6 +1005,7 @@ private MetadataDiff(StreamInput in) throws IOException { ); singleProject = null; + singleProjectReservedStateMetadata = null; multiProject = DiffableUtils.readJdkMapDiff( in, ProjectId.PROJECT_ID_SERIALIZER, @@ -1097,7 +1105,7 @@ public void writeTo(StreamOutput out) throws IOException { private Diff> buildUnifiedReservedStateMetadataDiff() { return DiffableUtils.merge( reservedStateMetadata, - singleProject.reservedStateMetadata(), + singleProjectReservedStateMetadata, DiffableUtils.getStringKeySerializer(), RESERVED_DIFF_VALUE_READER ); @@ -1268,6 +1276,10 @@ private static void readClusterCustoms(StreamInput in, Builder builder) throws I @Override public void writeTo(StreamOutput out) throws IOException { + writeTo(out, Collections.emptyMap()); + } + + public void writeTo(StreamOutput out, Map singleProjectReservedStateMetadata) throws IOException { out.writeLong(version); out.writeString(clusterUUID); out.writeBoolean(clusterUUIDCommitted); @@ -1305,10 +1317,10 @@ public void writeTo(StreamOutput out) throws IOException { VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms); List combinedMetadata = new ArrayList<>( - reservedStateMetadata.size() + singleProject.reservedStateMetadata().size() + reservedStateMetadata.size() + singleProjectReservedStateMetadata.size() ); combinedMetadata.addAll(reservedStateMetadata.values()); - combinedMetadata.addAll(singleProject.reservedStateMetadata().values()); + combinedMetadata.addAll(singleProjectReservedStateMetadata.values()); out.writeCollection(combinedMetadata); } else { VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 4b574b8313c28..344bbf402280d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -78,6 +78,7 @@ import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.elasticsearch.cluster.metadata.Metadata.ALL; +import static org.elasticsearch.cluster.project.ProjectStateRegistry.RESERVED_DIFF_VALUE_READER; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; public class ProjectMetadata implements Iterable, Diffable, ChunkedToXContent { @@ -91,7 +92,6 @@ public class ProjectMetadata implements Iterable, Diffable> aliasedIndices; private final ImmutableOpenMap templates; private final ImmutableOpenMap customs; - private final ImmutableOpenMap reservedStateMetadata; private final int totalNumberOfShards; private final int totalOpenIndexShards; @@ -125,7 +125,6 @@ private ProjectMetadata( ImmutableOpenMap> aliasedIndices, ImmutableOpenMap templates, ImmutableOpenMap customs, - ImmutableOpenMap reservedStateMetadata, int totalNumberOfShards, int totalOpenIndexShards, String[] allIndices, @@ -143,7 +142,6 @@ private ProjectMetadata( this.aliasedIndices = aliasedIndices; this.templates = templates; this.customs = customs; - this.reservedStateMetadata = reservedStateMetadata; this.totalNumberOfShards = totalNumberOfShards; this.totalOpenIndexShards = totalOpenIndexShards; this.allIndices = allIndices; @@ -224,7 +222,6 @@ public ProjectMetadata withLifecycleState(Index index, LifecycleExecutionState l aliasedIndices, templates, customs, - reservedStateMetadata, totalNumberOfShards, totalOpenIndexShards, allIndices, @@ -257,7 +254,6 @@ public ProjectMetadata withIndexSettingsUpdates(Map updates) { aliasedIndices, templates, customs, - reservedStateMetadata, totalNumberOfShards, totalOpenIndexShards, allIndices, @@ -291,7 +287,6 @@ public ProjectMetadata withAllocationAndTermUpdatesOnly(Map customs() { return customs; } - public Map reservedStateMetadata() { - return reservedStateMetadata; - } - public Map dataStreams() { return custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY).dataStreams(); } @@ -1143,7 +1133,6 @@ public static class Builder { private final ImmutableOpenMap.Builder indices; private final ImmutableOpenMap.Builder templates; private final ImmutableOpenMap.Builder customs; - private final ImmutableOpenMap.Builder reservedStateMetadata; private SortedMap previousIndicesLookup; @@ -1160,7 +1149,6 @@ public static class Builder { this.indices = ImmutableOpenMap.builder(projectMetadata.indices); this.templates = ImmutableOpenMap.builder(projectMetadata.templates); this.customs = ImmutableOpenMap.builder(projectMetadata.customs); - this.reservedStateMetadata = ImmutableOpenMap.builder(projectMetadata.reservedStateMetadata); this.previousIndicesLookup = projectMetadata.indicesLookup; this.mappingsByHash = new HashMap<>(projectMetadata.mappingsByHash); this.checkForUnusedMappings = false; @@ -1174,7 +1162,6 @@ public static class Builder { indices = ImmutableOpenMap.builder(indexCountHint); templates = ImmutableOpenMap.builder(); customs = ImmutableOpenMap.builder(); - reservedStateMetadata = ImmutableOpenMap.builder(); previousIndicesLookup = null; this.mappingsByHash = new HashMap<>(mappingsByHash); indexGraveyard(IndexGraveyard.builder().build()); // create new empty index graveyard to initialize @@ -1519,21 +1506,6 @@ public Builder customs(Map customs) { return this; } - public Builder put(Map reservedStateMetadata) { - this.reservedStateMetadata.putAllFromMap(reservedStateMetadata); - return this; - } - - public Builder put(ReservedStateMetadata metadata) { - reservedStateMetadata.put(metadata.namespace(), metadata); - return this; - } - - public Builder removeReservedState(ReservedStateMetadata metadata) { - reservedStateMetadata.remove(metadata.namespace()); - return this; - } - public Builder indexGraveyard(final IndexGraveyard indexGraveyard) { return putCustom(IndexGraveyard.TYPE, indexGraveyard); } @@ -1682,7 +1654,6 @@ public ProjectMetadata build(boolean skipNameCollisionChecks) { aliasedIndices, templates.build(), customs.build(), - reservedStateMetadata.build(), totalNumberOfShards, totalOpenIndexShards, allIndicesArray, @@ -2109,7 +2080,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep switch (currentFieldName) { case "reserved_state" -> { while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - projectBuilder.put(ReservedStateMetadata.fromXContent(parser)); + ReservedStateMetadata.fromXContent(parser); } } case "indices" -> { @@ -2169,10 +2140,7 @@ public Iterator toXContentChunked(ToXContent.Params p) { ) ), indices, - customs, - multiProject - ? ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator()) - : Collections.emptyIterator() + customs ); } @@ -2199,9 +2167,11 @@ public static ProjectMetadata readFrom(StreamInput in) throws IOException { readProjectCustoms(in, builder); - int reservedStateSize = in.readVInt(); - for (int i = 0; i < reservedStateSize; i++) { - builder.put(ReservedStateMetadata.readFrom(in)); + if (in.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + int reservedStateSize = in.readVInt(); + for (int i = 0; i < reservedStateSize; i++) { + ReservedStateMetadata.readFrom(in); + } } if (in.getTransportVersion() @@ -2238,7 +2208,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeCollection(templates.values()); VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); - out.writeCollection(reservedStateMetadata.values()); + if (out.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + out.writeCollection(Collections.emptySet()); + } if (out.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { @@ -2253,23 +2225,16 @@ static class ProjectMetadataDiff implements Diff { new DiffableUtils.DiffableValueReader<>(IndexMetadata::readFrom, IndexMetadata::readDiffFrom); private static final DiffableUtils.DiffableValueReader TEMPLATES_DIFF_VALUE_READER = new DiffableUtils.DiffableValueReader<>(IndexTemplateMetadata::readFrom, IndexTemplateMetadata::readDiffFrom); - private static final DiffableUtils.DiffableValueReader RESERVED_DIFF_VALUE_READER = - new DiffableUtils.DiffableValueReader<>(ReservedStateMetadata::readFrom, ReservedStateMetadata::readDiffFrom); private final DiffableUtils.MapDiff> indices; private final DiffableUtils.MapDiff> templates; private final DiffableUtils.MapDiff> customs; - private final DiffableUtils.MapDiff< - String, - ReservedStateMetadata, - ImmutableOpenMap> reservedStateMetadata; private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) { if (before == after) { indices = DiffableUtils.emptyDiff(); templates = DiffableUtils.emptyDiff(); customs = DiffableUtils.emptyDiff(); - reservedStateMetadata = DiffableUtils.emptyDiff(); } else { indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer()); @@ -2279,35 +2244,30 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) { DiffableUtils.getStringKeySerializer(), PROJECT_CUSTOM_VALUE_SERIALIZER ); - reservedStateMetadata = DiffableUtils.diff( - before.reservedStateMetadata, - after.reservedStateMetadata, - DiffableUtils.getStringKeySerializer() - ); } } ProjectMetadataDiff( DiffableUtils.MapDiff> indices, DiffableUtils.MapDiff> templates, - DiffableUtils.MapDiff> customs, - DiffableUtils.MapDiff> reservedStateMetadata + DiffableUtils.MapDiff> customs ) { this.indices = indices; this.templates = templates; this.customs = customs; - this.reservedStateMetadata = reservedStateMetadata; } ProjectMetadataDiff(StreamInput in) throws IOException { indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), INDEX_METADATA_DIFF_VALUE_READER); templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), TEMPLATES_DIFF_VALUE_READER); customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), PROJECT_CUSTOM_VALUE_SERIALIZER); - reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff( - in, - DiffableUtils.getStringKeySerializer(), - RESERVED_DIFF_VALUE_READER - ); + if (in.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + DiffableUtils.readImmutableOpenMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + RESERVED_DIFF_VALUE_READER + ); + } if (in.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { Settings.readSettingsDiffFromStream(in); @@ -2326,16 +2286,14 @@ DiffableUtils.MapDiff> reservedStateMetadata() { - return reservedStateMetadata; - } - @Override public void writeTo(StreamOutput out) throws IOException { indices.writeTo(out); templates.writeTo(out); customs.writeTo(out); - reservedStateMetadata.writeTo(out); + if (out.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + DiffableUtils.emptyDiff().writeTo(out); + } if (out.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { Settings.EMPTY_DIFF.writeTo(out); @@ -2344,7 +2302,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ProjectMetadata apply(ProjectMetadata part) { - if (indices.isEmpty() && templates.isEmpty() && customs.isEmpty() && reservedStateMetadata.isEmpty()) { + if (indices.isEmpty() && templates.isEmpty() && customs.isEmpty()) { // nothing to do return part; } @@ -2354,7 +2312,6 @@ public ProjectMetadata apply(ProjectMetadata part) { builder.indices(updatedIndices); builder.templates(templates.apply(part.templates)); builder.customs(customs.apply(part.customs)); - builder.put(reservedStateMetadata.apply(part.reservedStateMetadata)); if (part.indices == updatedIndices && builder.dataStreamMetadata() == part.custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY)) { builder.previousIndicesLookup = part.indicesLookup; diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index 014ee37724cbc..c12ccf5dc2215 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.NamedDiffable; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -28,6 +29,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -38,6 +40,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -46,7 +49,10 @@ public class ProjectStateRegistry extends AbstractNamedDiffable implements Custom, NamedDiffable { public static final String TYPE = "projects_registry"; public static final ProjectStateRegistry EMPTY = new ProjectStateRegistry(Collections.emptyMap(), Collections.emptySet(), 0); - private static final Entry EMPTY_ENTRY = new Entry(Settings.EMPTY); + private static final Entry EMPTY_ENTRY = new Entry(Settings.EMPTY, ImmutableOpenMap.of()); + + public static final DiffableUtils.DiffableValueReader RESERVED_DIFF_VALUE_READER = + new DiffableUtils.DiffableValueReader<>(ReservedStateMetadata::readFrom, ReservedStateMetadata::readDiffFrom); private final Map projectsEntries; // Projects that have been marked for deletion based on their file-based setting @@ -54,12 +60,17 @@ public class ProjectStateRegistry extends AbstractNamedDiffable implemen // A counter that is incremented each time one or more projects are marked for deletion. private final long projectsMarkedForDeletionGeneration; + public static ProjectStateRegistry get(ClusterState clusterState) { + return clusterState.custom(TYPE, EMPTY); + } + public ProjectStateRegistry(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_STATE_REGISTRY_ENTRY)) { projectsEntries = in.readMap(ProjectId::readFrom, Entry::readFrom); } else { Map settingsMap = in.readMap(ProjectId::readFrom, Settings::readSettingsFromStream); - projectsEntries = settingsMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new Entry(e.getValue()))); + projectsEntries = settingsMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, + e -> new Entry(e.getValue(), ImmutableOpenMap.of()))); } if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_STATE_REGISTRY_RECORDS_DELETIONS)) { projectsMarkedForDeletion = in.readCollectionAsImmutableSet(ProjectId::readFrom); @@ -80,6 +91,10 @@ private ProjectStateRegistry( this.projectsMarkedForDeletionGeneration = projectsMarkedForDeletionGeneration; } + public boolean hasProject(ProjectId projectId) { + return projectsEntries.containsKey(projectId); + } + /** * Retrieves the settings for a specific project based on its project ID from the specified cluster state without creating a new object. * If you need a full state of the project rather than just its setting, please use {@link ClusterState#projectState(ProjectId)} @@ -97,6 +112,10 @@ public Settings getProjectSettings(ProjectId projectId) { return projectsEntries.getOrDefault(projectId, EMPTY_ENTRY).settings; } + public Map reservedStateMetadata(ProjectId projectId) { + return projectsEntries.getOrDefault(projectId, EMPTY_ENTRY).reservedStateMetadata; + } + public boolean isProjectMarkedForDeletion(ProjectId projectId) { return projectsMarkedForDeletion.contains(projectId); } @@ -292,14 +311,22 @@ private Builder(ProjectStateRegistry original) { this.projectsMarkedForDeletionGeneration = original.projectsMarkedForDeletionGeneration; } - public Builder putProjectSettings(ProjectId projectId, Settings settings) { + private void modifyEntry(ProjectId projectId, Function modifier) { Entry entry = projectsEntries.get(projectId); if (entry == null) { - entry = new Entry(settings); - } else { - entry = entry.withSettings(settings); + entry = new Entry(); } + entry = modifier.apply(entry); projectsEntries.put(projectId, entry); + } + + public Builder putProjectSettings(ProjectId projectId, Settings settings) { + modifyEntry(projectId, entry -> entry.withSettings(settings)); + return this; + } + + public Builder putReservedStateMetadata(ProjectId projectId, ReservedStateMetadata reservedStateMetadata) { + modifyEntry(projectId, entry -> entry.withReservedStateMetadata(reservedStateMetadata)); return this; } @@ -325,25 +352,55 @@ public ProjectStateRegistry build() { } } - private record Entry(Settings settings) implements Writeable, Diffable { + private record Entry(Settings settings, + ImmutableOpenMap reservedStateMetadata) implements Writeable, Diffable { + Entry() { + this(Settings.EMPTY, ImmutableOpenMap.of()); + } public static Entry readFrom(StreamInput in) throws IOException { - return new Entry(Settings.readSettingsFromStream(in)); + Settings settings = Settings.readSettingsFromStream(in); + + ImmutableOpenMap reservedStateMetadata; + if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + int reservedStateSize = in.readVInt(); + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(reservedStateSize); + for (int i = 0; i < reservedStateSize; i++) { + ReservedStateMetadata r = ReservedStateMetadata.readFrom(in); + builder.put(r.namespace(), r); + } + reservedStateMetadata = builder.build(); + } else { + reservedStateMetadata = ImmutableOpenMap.of(); + } + + return new Entry(settings, reservedStateMetadata); } public Entry withSettings(Settings settings) { - return new Entry(settings); + return new Entry(settings, reservedStateMetadata); + } + + public Entry withReservedStateMetadata(ReservedStateMetadata reservedStateMetadata) { + ImmutableOpenMap build = ImmutableOpenMap.builder(this.reservedStateMetadata) + .fPut(reservedStateMetadata.namespace(), reservedStateMetadata) + .build(); + return new Entry(settings, build); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeWriteable(settings); + if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + out.writeCollection(reservedStateMetadata.values()); + } } public void toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject("settings"); settings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); builder.endObject(); + ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator()); } @Override @@ -351,21 +408,43 @@ public Diff diff(Entry previousState) { if (this == previousState) { return SimpleDiffable.empty(); } - return new EntryDiff(settings.diff(previousState.settings)); + + return new EntryDiff(settings.diff(previousState.settings), DiffableUtils.diff( + previousState.reservedStateMetadata, + reservedStateMetadata, + DiffableUtils.getStringKeySerializer() + )); } - private record EntryDiff(Diff settingsDiff) implements Diff { + private record EntryDiff(Diff settingsDiff, DiffableUtils.MapDiff< + String, + ReservedStateMetadata, + ImmutableOpenMap> reservedStateMetadata) implements Diff { + public static EntryDiff readFrom(StreamInput in) throws IOException { - return new EntryDiff(Settings.readSettingsDiffFromStream(in)); + DiffableUtils.MapDiff> reservedStateMetadata; + if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + RESERVED_DIFF_VALUE_READER + ); + } else { + reservedStateMetadata = DiffableUtils.emptyDiff(); + } + return new EntryDiff(Settings.readSettingsDiffFromStream(in), reservedStateMetadata); } @Override public Entry apply(Entry part) { - return part.withSettings(settingsDiff.apply(part.settings)); + return new Entry(settingsDiff.apply(part.settings), reservedStateMetadata.apply(part.reservedStateMetadata)); } @Override public void writeTo(StreamOutput out) throws IOException { + if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { + reservedStateMetadata.writeTo(out); + } out.writeWriteable(settingsDiff); } } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index f6f2131f79d3d..e3479ee749c0d 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -125,11 +125,13 @@ public Path watchedFile() { *

* If there's no file based settings file in this cluster, we'll remove all state reservations for * file based settings from the cluster state. + * * @param clusterState the cluster state before snapshot restore - * @param mdBuilder the current metadata builder for the new cluster state - * @param projectId the project associated with the restore + * @param builder the current ClusterState builder for the new cluster state + * @param mdBuilder the current metadata builder for the new cluster state + * @param projectId the project associated with the restore */ - public void handleSnapshotRestore(ClusterState clusterState, Metadata.Builder mdBuilder, ProjectId projectId) { + public void handleSnapshotRestore(ClusterState clusterState, ClusterState.Builder builder, Metadata.Builder mdBuilder, ProjectId projectId) { assert clusterState.nodes().isLocalNodeElectedMaster(); ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(NAMESPACE); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java index 0e0320e258b6c..b081741a0e426 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedClusterStateService.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; @@ -451,7 +452,7 @@ public void process( ProjectMetadata projectMetadata = getPotentiallyNewProject(state, projectId); state = ClusterState.builder(state).putProjectMetadata(projectMetadata).build(); - ReservedStateMetadata existingMetadata = projectMetadata.reservedStateMetadata().get(namespace); + ReservedStateMetadata existingMetadata = ProjectStateRegistry.get(state).reservedStateMetadata(projectId).get(namespace); // We check if we should exit early on the state version from clusterService. The ReservedStateUpdateTask // will check again with the most current state version if this continues. diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java index db564ed2a838c..5c6900be83377 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java @@ -65,7 +65,7 @@ protected ClusterState execute(ClusterState currentState) { ProjectMetadata currentProject = ReservedClusterStateService.getPotentiallyNewProject(currentState, projectId); var result = execute( ClusterState.builder(currentState).putProjectMetadata(currentProject).build(), - currentProject.reservedStateMetadata() + ProjectStateRegistry.get(currentState).reservedStateMetadata(projectId) ); if (result == null) { return currentState; @@ -78,8 +78,8 @@ protected ClusterState execute(ClusterState currentState) { ); ProjectMetadata updatedProjectMetadata = updatedClusterState.getMetadata().getProject(projectId); return ClusterState.builder(currentState) - .putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(updatedProjectStateRegistry).build()) - .putProjectMetadata(ProjectMetadata.builder(updatedProjectMetadata).put(result.v2())) + .putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(updatedProjectStateRegistry).putReservedStateMetadata(projectId, result.v2()).build()) + .putProjectMetadata(ProjectMetadata.builder(updatedProjectMetadata)) .build(); } } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java index 225e491013606..cb4478a43ad41 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java @@ -16,9 +16,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import static org.elasticsearch.cluster.metadata.ReservedStateMetadata.EMPTY_VERSION; import static org.elasticsearch.cluster.metadata.ReservedStateMetadata.NO_VERSION; @@ -63,7 +64,7 @@ static boolean isNewError(ReservedStateMetadata existingMetadata, Long newStateV static ReservedStateMetadata getMetadata(ClusterState state, ErrorState errorState) { return errorState.projectId() - .map(p -> ReservedClusterStateService.getPotentiallyNewProject(state, p).reservedStateMetadata()) + .map(ProjectStateRegistry.get(state)::reservedStateMetadata) .orElseGet(() -> state.metadata().reservedStateMetadata()) .get(errorState.namespace()); } @@ -93,13 +94,16 @@ ClusterState execute(ClusterState currentState) { var errorMetadata = new ReservedStateErrorMetadata(errorState.version(), errorState.errorKind(), errorState.errors()); if (errorState.projectId().isPresent()) { - ProjectMetadata project = currentState.metadata().getProject(errorState.projectId().get()); + ProjectStateRegistry projectStateRegistry = currentState.custom(ProjectStateRegistry.TYPE); - ReservedStateMetadata reservedMetadata = project.reservedStateMetadata().get(errorState.namespace()); + ProjectId projectId = errorState.projectId().get(); + ReservedStateMetadata reservedMetadata = projectStateRegistry.reservedStateMetadata(projectId).get(errorState.namespace()); ReservedStateMetadata.Builder resBuilder = ReservedStateMetadata.builder(errorState.namespace(), reservedMetadata); resBuilder.errorMetadata(errorMetadata); - stateBuilder.putProjectMetadata(ProjectMetadata.builder(project).put(resBuilder.build())); + stateBuilder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(projectStateRegistry) + .putReservedStateMetadata(projectId, resBuilder.build()) + .build()); } else { Metadata.Builder metadataBuilder = Metadata.builder(currentState.metadata()); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 23e9f23a3e46b..752fafabee3bb 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1567,7 +1567,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) { // Restore global state if needed if (request.includeGlobalState()) { applyGlobalStateRestore(currentState, mdBuilder, projectId); - fileSettingsService.handleSnapshotRestore(currentState, mdBuilder, projectId); + fileSettingsService.handleSnapshotRestore(currentState, builder, mdBuilder, projectId); } if (completed(shards)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index cb9fa23aaefbc..3cb50b17b2272 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; @@ -754,8 +755,8 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { var updatedState = processJSON(action, prevState, settingsJSON); - ProjectMetadata withReservedState = ProjectMetadata.builder(updatedState.state().getMetadata().getProject(projectId)) - .put( + ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()).putReservedStateMetadata( + projectId, ReservedStateMetadata.builder("test") .putHandler(new ReservedStateHandlerMetadata(ReservedComposableIndexTemplateAction.NAME, updatedState.keys())) .build() @@ -810,7 +811,7 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { IllegalArgumentException.class, () -> TransportPutComposableIndexTemplateAction.verifyIfUsingReservedComponentTemplates( request, - withReservedState.reservedStateMetadata().values() + withReservedState.reservedStateMetadata(projectId).values() ) ).getMessage().contains("errors: [[component_template:template_1] is reserved by [test]]") ); @@ -824,7 +825,7 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { // this should just work, no failure TransportPutComposableIndexTemplateAction.verifyIfUsingReservedComponentTemplates( request, - withReservedState.reservedStateMetadata().values() + withReservedState.reservedStateMetadata(projectId).values() ); } } @@ -922,8 +923,8 @@ public void testTemplatesWithReservedPrefix() throws Exception { allOf(aMapWithSize(2), hasKey(reservedComposableIndexName(conflictingTemplateName)), hasKey(conflictingTemplateName)) ); - ProjectMetadata withReservedMetadata = ProjectMetadata.builder(updatedState.state().getMetadata().getProject(projectId)) - .put( + ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()).putReservedStateMetadata( + projectId, new ReservedStateMetadata.Builder("file_settings").putHandler( new ReservedStateHandlerMetadata(ReservedComposableIndexTemplateAction.NAME, updatedState.keys()) ).build() @@ -957,7 +958,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { expectThrows( IllegalArgumentException.class, () -> fakeAction.validateForReservedState( - withReservedMetadata.reservedStateMetadata().values(), + withReservedState.reservedStateMetadata(projectId).values(), ReservedComposableIndexTemplateAction.NAME, modifiedKeys, pr.name() @@ -973,7 +974,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { assertThat(modifiedKeysOK, hasSize(1)); fakeAction.validateForReservedState( - withReservedMetadata.reservedStateMetadata().values(), + withReservedState.reservedStateMetadata(projectId).values(), ReservedComposableIndexTemplateAction.NAME, modifiedKeysOK, prOK.name() diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 373831b54804b..3419b30bd2e89 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -969,10 +969,6 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata .sum(); int reservedStateSize = metadata.reservedStateMetadata().size(); - if (params.paramAsBoolean("multi-project", false) == false) { - // only one project if not multi-project, add its reserved state to the cluster's collection - reservedStateSize += metadata.projects().values().iterator().next().reservedStateMetadata().size(); - } // 2 chunks for wrapping reserved state + 1 chunk for each item chunkCount += 2 + reservedStateSize; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index bd63b6e70371d..8fd7089b8ecbe 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -2849,11 +2849,6 @@ static int expectedChunkCount(ToXContent.Params params, ProjectMetadata project) } } - if (params.paramAsBoolean("multi-project", false)) { - // 2 chunks for wrapping reserved state + 1 chunk for each item - chunkCount += 2 + project.reservedStateMetadata().size(); - } - return Math.toIntExact(chunkCount); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index e10012133ffcd..f6949c181ca12 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -127,8 +127,6 @@ public void testSimpleJsonFromAndTo() throws IOException { .put(idx2, false) .put(DataStreamTestHelper.newInstance("data-stream1", List.of(idx1.getIndex()))) .put(DataStreamTestHelper.newInstance("data-stream2", List.of(idx2.getIndex()))) - .put(reservedStateMetadata) - .put(reservedStateMetadata1) .build(); XContentBuilder builder = JsonXContent.contentBuilder(); diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index 7e8e262f954dc..ddf1a81579e0a 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -453,7 +453,7 @@ public void testHandleSnapshotRestoreClearsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(state.metadata()); - fileSettingsService.handleSnapshotRestore(state, metadata, ProjectId.DEFAULT); + fileSettingsService.handleSnapshotRestore(state, builder, metadata, ProjectId.DEFAULT); assertThat(metadata.build().reservedStateMetadata(), anEmptyMap()); } @@ -476,7 +476,7 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(); - fileSettingsService.handleSnapshotRestore(state, metadata, ProjectId.DEFAULT); + fileSettingsService.handleSnapshotRestore(state, builder, metadata, ProjectId.DEFAULT); assertThat( metadata.build().reservedStateMetadata(), diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java index 981dd5e9475a2..75e3a73d09328 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata; import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; import org.elasticsearch.cluster.metadata.ReservedStateMetadata; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.RerouteService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; @@ -183,7 +184,7 @@ private static ClusterState setupProject(ClusterState state, Optional } private static Map getMetadata(ClusterState state, Optional projectId) { - return projectId.map(p -> state.metadata().getProject(p).reservedStateMetadata()) + return projectId.map(p -> ProjectStateRegistry.get(state).reservedStateMetadata(p)) .orElseGet(() -> state.metadata().reservedStateMetadata()); } @@ -739,10 +740,13 @@ public void testUpdateTaskDuplicateError() { Optional projectId = randomBoolean() ? Optional.empty() : Optional.of(randomProjectIdOrDefault()); - Metadata metadata = projectId.map(p -> Metadata.builder().put(ProjectMetadata.builder(p).put(operatorMetadata))) - .orElseGet(() -> Metadata.builder().put(operatorMetadata)) - .build(); - ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); + ClusterState.Builder builder = ClusterState.builder(new ClusterName("test")); + if (projectId.isPresent()) { + builder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build()); + } else { + builder.metadata(Metadata.builder().put(operatorMetadata)); + } + ClusterState state = builder.build(); assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 2L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); assertFalse(ReservedStateErrorTask.isNewError(operatorMetadata, 1L, ReservedStateVersionCheck.HIGHER_VERSION_ONLY)); @@ -849,10 +853,14 @@ public TransformState transform(Map source, TransformState prevS .putHandler(hmOne) .build(); - metadata = projectId.map(p -> Metadata.builder().put(ProjectMetadata.builder(p).put(opMetadata))) - .orElseGet(() -> Metadata.builder().put(opMetadata)) - .build(); - ClusterState newState = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); + + builder = ClusterState.builder(new ClusterName("test")); + if (projectId.isPresent()) { + builder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), opMetadata).build()); + } else { + builder.metadata(Metadata.builder().put(opMetadata)); + } + ClusterState newState = builder.build(); // We exit on duplicate errors before we update the reserved state error metadata assertThat( @@ -865,10 +873,13 @@ public void testCheckMetadataVersion() { ReservedStateMetadata operatorMetadata = ReservedStateMetadata.builder("test").version(123L).build(); Optional projectId = randomBoolean() ? Optional.empty() : Optional.of(randomProjectIdOrDefault()); - Metadata metadata = projectId.map(p -> Metadata.builder().put(ProjectMetadata.builder(p).put(operatorMetadata))) - .orElseGet(() -> Metadata.builder().put(operatorMetadata)) - .build(); - ClusterState state = ClusterState.builder(new ClusterName("test")).metadata(metadata).build(); + ClusterState.Builder builder = ClusterState.builder(new ClusterName("test")); + if (projectId.isPresent()) { + builder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build()); + } else { + builder.metadata(Metadata.builder().put(operatorMetadata)); + } + ClusterState state = builder.build(); ReservedStateUpdateTask task = createEmptyTask( projectId, From d136933726a1b3a21ae53c6eb84dc4e377b11e6c Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 1 Aug 2025 14:57:58 +0100 Subject: [PATCH 02/16] Change the order of fields in a serialised version for consistency --- .../cluster/project/ProjectStateRegistry.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index c12ccf5dc2215..5a4ab8cef2f19 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -422,6 +422,8 @@ private record EntryDiff(Diff settingsDiff, DiffableUtils.MapDiff< ImmutableOpenMap> reservedStateMetadata) implements Diff { public static EntryDiff readFrom(StreamInput in) throws IOException { + Diff settingsDiff = Settings.readSettingsDiffFromStream(in); + DiffableUtils.MapDiff> reservedStateMetadata; if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff( @@ -432,7 +434,8 @@ public static EntryDiff readFrom(StreamInput in) throws IOException { } else { reservedStateMetadata = DiffableUtils.emptyDiff(); } - return new EntryDiff(Settings.readSettingsDiffFromStream(in), reservedStateMetadata); + + return new EntryDiff(settingsDiff, reservedStateMetadata); } @Override @@ -442,10 +445,10 @@ public Entry apply(Entry part) { @Override public void writeTo(StreamOutput out) throws IOException { + out.writeWriteable(settingsDiff); if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { reservedStateMetadata.writeTo(out); } - out.writeWriteable(settingsDiff); } } } From 6088f4a23a99a78f9bf0f4c288709b1bb80127cf Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 1 Aug 2025 14:17:00 +0000 Subject: [PATCH 03/16] [CI] Auto commit changes from spotless --- ...sportPutComposableIndexTemplateAction.java | 5 ++- .../elasticsearch/cluster/ClusterState.java | 15 ++++---- .../cluster/metadata/Metadata.java | 13 ++++--- .../cluster/metadata/ProjectMetadata.java | 6 +--- .../cluster/project/ProjectStateRegistry.java | 35 ++++++++++--------- .../service/FileSettingsService.java | 7 +++- .../ReservedProjectStateUpdateTask.java | 5 ++- .../service/ReservedStateErrorTask.java | 7 ++-- ...vedComposableIndexTemplateActionTests.java | 6 ++-- .../ReservedClusterStateServiceTests.java | 16 ++++++--- 10 files changed, 71 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index 9c18078ada71b..2ca82a3ddbd68 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -88,7 +88,10 @@ protected void masterOperation( ) { ProjectId projectId = projectResolver.getProjectId(); verifyIfUsingReservedComponentTemplates(request, state.metadata().reservedStateMetadata().values()); - verifyIfUsingReservedComponentTemplates(request, ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values()); + verifyIfUsingReservedComponentTemplates( + request, + ProjectStateRegistry.get(state).reservedStateMetadata(projectResolver.getProjectId()).values() + ); ComposableIndexTemplate indexTemplate = request.indexTemplate(); indexTemplateService.putIndexTemplateV2( request.cause(), diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index c1d3dbbae83a5..9defaa01a639c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -1320,7 +1320,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeString(stateUUID); if (out.getTransportVersion().before(TransportVersions.MULTI_PROJECT)) { - Map singleProjectReservedState = ProjectStateRegistry.get(this).reservedStateMetadata(Metadata.DEFAULT_PROJECT_ID); + Map singleProjectReservedState = ProjectStateRegistry.get(this) + .reservedStateMetadata(Metadata.DEFAULT_PROJECT_ID); metadata.writeTo(out, singleProjectReservedState); } else { metadata.writeTo(out); @@ -1384,12 +1385,12 @@ private Diff getMetadataDiff(ClusterState before, ClusterState after) final Diff metadata; ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(after); if (projectStateRegistry.size() == 1 && projectStateRegistry.hasProject(Metadata.DEFAULT_PROJECT_ID)) { - Map reservedStateMetadataBefore = ProjectStateRegistry.get(before).reservedStateMetadata(DEFAULT_PROJECT_ID); - Map reservedStateMetadataAfter = projectStateRegistry.reservedStateMetadata(DEFAULT_PROJECT_ID); - DiffableUtils.MapDiff< - String, - ReservedStateMetadata, - Map> diff = DiffableUtils.diff( + Map reservedStateMetadataBefore = ProjectStateRegistry.get(before) + .reservedStateMetadata(DEFAULT_PROJECT_ID); + Map reservedStateMetadataAfter = projectStateRegistry.reservedStateMetadata( + DEFAULT_PROJECT_ID + ); + DiffableUtils.MapDiff> diff = DiffableUtils.diff( reservedStateMetadataBefore, reservedStateMetadataAfter, DiffableUtils.getStringKeySerializer() diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 5fcf20575b4cd..4b52fd9ddc2a4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -709,8 +709,10 @@ public Diff diff(Metadata previousState) { return diff(previousState, DiffableUtils.emptyDiff()); } - public Diff diff(Metadata previousState, - MapDiff> singleProjectReservedStateMetadata) { + public Diff diff( + Metadata previousState, + MapDiff> singleProjectReservedStateMetadata + ) { return new MetadataDiff(previousState, this, singleProjectReservedStateMetadata); } @@ -864,8 +866,11 @@ private static class MetadataDiff implements Diff { // This is used only when the node has a single project and needs to send the diff to an old node (wire BWC). private final MapDiff> combinedTasksDiff; - MetadataDiff(Metadata before, Metadata after, - MapDiff> singleProjectReservedState) { + MetadataDiff( + Metadata before, + Metadata after, + MapDiff> singleProjectReservedState + ) { this.empty = before == after; this.fromNodeBeforeMultiProjectsSupport = false; // diff on this node, always after multi-projects, even when disabled clusterUUID = after.clusterUUID; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 344bbf402280d..5f7df4daed4bd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -2262,11 +2262,7 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) { templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), TEMPLATES_DIFF_VALUE_READER); customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), PROJECT_CUSTOM_VALUE_SERIALIZER); if (in.getTransportVersion().before(TransportVersions.PROJECT_RESERVED_STATE_MOVE_TO_REGISTRY)) { - DiffableUtils.readImmutableOpenMapDiff( - in, - DiffableUtils.getStringKeySerializer(), - RESERVED_DIFF_VALUE_READER - ); + DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), RESERVED_DIFF_VALUE_READER); } if (in.getTransportVersion() .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index d582abec3e4dd..c84267f1fc626 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -69,8 +69,9 @@ public ProjectStateRegistry(StreamInput in) throws IOException { projectsEntries = in.readMap(ProjectId::readFrom, Entry::readFrom); } else { Map settingsMap = in.readMap(ProjectId::readFrom, Settings::readSettingsFromStream); - projectsEntries = settingsMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, - e -> new Entry(e.getValue(), ImmutableOpenMap.of()))); + projectsEntries = settingsMap.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new Entry(e.getValue(), ImmutableOpenMap.of()))); } if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_STATE_REGISTRY_RECORDS_DELETIONS)) { projectsMarkedForDeletion = in.readCollectionAsImmutableSet(ProjectId::readFrom); @@ -115,7 +116,7 @@ public Settings getProjectSettings(ProjectId projectId) { public Map reservedStateMetadata(ProjectId projectId) { return projectsEntries.getOrDefault(projectId, EMPTY_ENTRY).reservedStateMetadata; } - + public Set getProjectsMarkedForDeletion() { return projectsMarkedForDeletion; } @@ -362,11 +363,14 @@ public ProjectStateRegistry build() { } } - private record Entry(Settings settings, - ImmutableOpenMap reservedStateMetadata) implements Writeable, Diffable { - Entry() { + private record Entry(Settings settings, ImmutableOpenMap reservedStateMetadata) + implements + Writeable, + Diffable { + + Entry() { this(Settings.EMPTY, ImmutableOpenMap.of()); - } + } public static Entry readFrom(StreamInput in) throws IOException { Settings settings = Settings.readSettingsFromStream(in); @@ -419,17 +423,16 @@ public Diff diff(Entry previousState) { return SimpleDiffable.empty(); } - return new EntryDiff(settings.diff(previousState.settings), DiffableUtils.diff( - previousState.reservedStateMetadata, - reservedStateMetadata, - DiffableUtils.getStringKeySerializer() - )); + return new EntryDiff( + settings.diff(previousState.settings), + DiffableUtils.diff(previousState.reservedStateMetadata, reservedStateMetadata, DiffableUtils.getStringKeySerializer()) + ); } - private record EntryDiff(Diff settingsDiff, DiffableUtils.MapDiff< - String, - ReservedStateMetadata, - ImmutableOpenMap> reservedStateMetadata) implements Diff { + private record EntryDiff( + Diff settingsDiff, + DiffableUtils.MapDiff> reservedStateMetadata + ) implements Diff { public static EntryDiff readFrom(StreamInput in) throws IOException { Diff settingsDiff = Settings.readSettingsDiffFromStream(in); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index e3479ee749c0d..2eaf50ca3849f 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -131,7 +131,12 @@ public Path watchedFile() { * @param mdBuilder the current metadata builder for the new cluster state * @param projectId the project associated with the restore */ - public void handleSnapshotRestore(ClusterState clusterState, ClusterState.Builder builder, Metadata.Builder mdBuilder, ProjectId projectId) { + public void handleSnapshotRestore( + ClusterState clusterState, + ClusterState.Builder builder, + Metadata.Builder mdBuilder, + ProjectId projectId + ) { assert clusterState.nodes().isLocalNodeElectedMaster(); ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(NAMESPACE); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java index 5c6900be83377..a1633009ffde1 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java @@ -78,7 +78,10 @@ protected ClusterState execute(ClusterState currentState) { ); ProjectMetadata updatedProjectMetadata = updatedClusterState.getMetadata().getProject(projectId); return ClusterState.builder(currentState) - .putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(updatedProjectStateRegistry).putReservedStateMetadata(projectId, result.v2()).build()) + .putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder(updatedProjectStateRegistry).putReservedStateMetadata(projectId, result.v2()).build() + ) .putProjectMetadata(ProjectMetadata.builder(updatedProjectMetadata)) .build(); } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java index cb4478a43ad41..bf59e2b96d7fc 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java @@ -101,9 +101,10 @@ ClusterState execute(ClusterState currentState) { ReservedStateMetadata.Builder resBuilder = ReservedStateMetadata.builder(errorState.namespace(), reservedMetadata); resBuilder.errorMetadata(errorMetadata); - stateBuilder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(projectStateRegistry) - .putReservedStateMetadata(projectId, resBuilder.build()) - .build()); + stateBuilder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder(projectStateRegistry).putReservedStateMetadata(projectId, resBuilder.build()).build() + ); } else { Metadata.Builder metadataBuilder = Metadata.builder(currentState.metadata()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 3cb50b17b2272..8513cf5eff45f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -755,7 +755,8 @@ public void testBlockUsingReservedComponentTemplates() throws Exception { var updatedState = processJSON(action, prevState, settingsJSON); - ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()).putReservedStateMetadata( + ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()) + .putReservedStateMetadata( projectId, ReservedStateMetadata.builder("test") .putHandler(new ReservedStateHandlerMetadata(ReservedComposableIndexTemplateAction.NAME, updatedState.keys())) @@ -923,7 +924,8 @@ public void testTemplatesWithReservedPrefix() throws Exception { allOf(aMapWithSize(2), hasKey(reservedComposableIndexName(conflictingTemplateName)), hasKey(conflictingTemplateName)) ); - ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()).putReservedStateMetadata( + ProjectStateRegistry withReservedState = ProjectStateRegistry.builder(updatedState.state()) + .putReservedStateMetadata( projectId, new ReservedStateMetadata.Builder("file_settings").putHandler( new ReservedStateHandlerMetadata(ReservedComposableIndexTemplateAction.NAME, updatedState.keys()) diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java index 75e3a73d09328..ac38d3e1935be 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/ReservedClusterStateServiceTests.java @@ -742,7 +742,10 @@ public void testUpdateTaskDuplicateError() { ClusterState.Builder builder = ClusterState.builder(new ClusterName("test")); if (projectId.isPresent()) { - builder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build()); + builder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build() + ); } else { builder.metadata(Metadata.builder().put(operatorMetadata)); } @@ -853,10 +856,12 @@ public TransformState transform(Map source, TransformState prevS .putHandler(hmOne) .build(); - builder = ClusterState.builder(new ClusterName("test")); if (projectId.isPresent()) { - builder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), opMetadata).build()); + builder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), opMetadata).build() + ); } else { builder.metadata(Metadata.builder().put(opMetadata)); } @@ -875,7 +880,10 @@ public void testCheckMetadataVersion() { Optional projectId = randomBoolean() ? Optional.empty() : Optional.of(randomProjectIdOrDefault()); ClusterState.Builder builder = ClusterState.builder(new ClusterName("test")); if (projectId.isPresent()) { - builder.putCustom(ProjectStateRegistry.TYPE, ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build()); + builder.putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder().putReservedStateMetadata(projectId.get(), operatorMetadata).build() + ); } else { builder.metadata(Metadata.builder().put(operatorMetadata)); } From 84b0c9e74f78927e9f4077b427f0d3ccb69af2a7 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 1 Aug 2025 15:26:25 +0100 Subject: [PATCH 04/16] Fix build --- .../reservedstate/service/FileSettingsServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java index ddf1a81579e0a..987da3c95805d 100644 --- a/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java @@ -453,7 +453,7 @@ public void testHandleSnapshotRestoreClearsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(state.metadata()); - fileSettingsService.handleSnapshotRestore(state, builder, metadata, ProjectId.DEFAULT); + fileSettingsService.handleSnapshotRestore(state, ClusterState.builder(state), metadata, ProjectId.DEFAULT); assertThat(metadata.build().reservedStateMetadata(), anEmptyMap()); } @@ -476,7 +476,7 @@ public void testHandleSnapshotRestoreResetsMetadata() throws Exception { .build(); Metadata.Builder metadata = Metadata.builder(); - fileSettingsService.handleSnapshotRestore(state, builder, metadata, ProjectId.DEFAULT); + fileSettingsService.handleSnapshotRestore(state, ClusterState.builder(state), metadata, ProjectId.DEFAULT); assertThat( metadata.build().reservedStateMetadata(), From a18c3902afe5daa08913ad078e60708fc7978e9d Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 1 Aug 2025 18:08:30 +0100 Subject: [PATCH 05/16] Fix tests --- .../cluster/metadata/Metadata.java | 8 +- .../service/ReservedStateErrorTask.java | 2 +- .../cluster/ClusterStateTests.java | 76 +++++++++++++++++-- .../metadata/ProjectMetadataTests.java | 3 +- .../metadata/ToAndFromJsonMetadataTests.java | 5 +- 5 files changed, 81 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 4b52fd9ddc2a4..458b305b218d5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -824,13 +824,19 @@ private Iterator toXContentChunkedWithSingleProjectFormat( ); } + // make order deterministic + Iterator reservedStateMetadataIterator = reservedStateMetadata.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue) + .iterator(); + return Iterators.concat( start, clusterCoordination, persistentSettings, project.toXContentChunked(p), customs, - ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata.values().iterator()), + ChunkedToXContentHelper.object("reserved_state", reservedStateMetadataIterator), ChunkedToXContentHelper.endObject() ); } diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java index bf59e2b96d7fc..3113505561e46 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedStateErrorTask.java @@ -94,7 +94,7 @@ ClusterState execute(ClusterState currentState) { var errorMetadata = new ReservedStateErrorMetadata(errorState.version(), errorState.errorKind(), errorState.errors()); if (errorState.projectId().isPresent()) { - ProjectStateRegistry projectStateRegistry = currentState.custom(ProjectStateRegistry.TYPE); + ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(currentState); ProjectId projectId = errorState.projectId().get(); ReservedStateMetadata reservedMetadata = projectStateRegistry.reservedStateMetadata(projectId).get(errorState.namespace()); diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 78623dd3f2738..a5e597f5bc94a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -26,6 +26,8 @@ import org.elasticsearch.cluster.metadata.MetadataTests; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -44,6 +46,9 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -85,8 +90,12 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -515,8 +524,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "event_ingested_range": { "shards": [] } } }, - "index-graveyard": { "tombstones": [] }, - "reserved_state": {} + "index-graveyard": { "tombstones": [] } }, { "id": "3LftaL7hgfXAsF60Gm6jcD", @@ -573,15 +581,13 @@ public void testToXContentWithMultipleProjects() throws IOException { "event_ingested_range": { "shards": [] } } }, - "index-graveyard": { "tombstones": [] }, - "reserved_state": {} + "index-graveyard": { "tombstones": [] } }, { "id": "WHyuJ0uqBYOPgHX9kYUXlZ", "templates": {}, "indices": {}, - "index-graveyard": { "tombstones": [] }, - "reserved_state": {} + "index-graveyard": { "tombstones": [] } } ], "reserved_state": {} @@ -2241,4 +2247,62 @@ public static int expectedChunkCount(ToXContent.Params params, ClusterState clus return Math.toIntExact(chunkCount); } + + public void testSerialization() throws IOException { + ClusterState clusterState = buildClusterState(); + BytesStreamOutput out = new BytesStreamOutput(); + clusterState.writeTo(out); + + // check it deserializes ok + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + ClusterState deserialisedClusterState = ClusterState.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry), null); + + // check it matches the original object + Metadata deserializedMetadata = deserialisedClusterState.metadata(); + assertThat(deserializedMetadata.projects(), aMapWithSize(1)); + assertThat(deserializedMetadata.projects(), hasKey(ProjectId.DEFAULT)); + + assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).templates(), hasKey("template")); + assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).indices(), hasKey("index")); + } + + public void testCombinedReservedMetadataSerialization() throws IOException { + ClusterState clusterState = ClusterState.builder(buildClusterState()) + .putCustom( + ProjectStateRegistry.TYPE, + ProjectStateRegistry.builder() + .putReservedStateMetadata(ProjectId.DEFAULT, ReservedStateMetadata.builder("file_settings").putHandler( + new ReservedStateHandlerMetadata("settings", Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey())) + ).build()) + .build() + ) + .build(); + + TransportVersion oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT); + + BytesStreamOutput out = new BytesStreamOutput(); + out.setTransportVersion(oldVersion); + clusterState.writeTo(out); + + // check it deserializes ok + NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry); + in.setTransportVersion(oldVersion); + ClusterState deserialisedClusterState = ClusterState.readFrom(in, null); + + // check it matches the original object + Metadata deserializedMetadata = deserialisedClusterState.metadata(); + assertThat(deserializedMetadata.projects(), aMapWithSize(1)); + assertThat(deserializedMetadata.projects(), hasKey(ProjectId.DEFAULT)); + + assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).templates(), hasKey("template")); + assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).indices(), hasKey("index")); + + assertThat(deserializedMetadata.reservedStateMetadata(), hasKey("file_settings")); + ReservedStateMetadata fileSettings = deserializedMetadata.reservedStateMetadata().get("file_settings"); + assertThat(fileSettings.handlers(), aMapWithSize(1)); + assertThat(fileSettings.handlers(), hasKey("settings")); + ReservedStateHandlerMetadata settingsHandlerMetadata = fileSettings.handlers().get("settings"); + assertThat(settingsHandlerMetadata.keys(), equalTo(Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey()))); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index 8fd7089b8ecbe..71267cb3d7e33 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -2745,8 +2745,7 @@ public void testToXContentMultiProject() throws IOException { } }, "data_stream_aliases": {} - }, - "reserved_state": {} + } } """, IndexVersion.current(), diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index f6949c181ca12..b133bbde7552a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -131,7 +131,7 @@ public void testSimpleJsonFromAndTo() throws IOException { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); - ChunkedToXContent.wrapAsToXContent(Metadata.builder().put(project).build()) + ChunkedToXContent.wrapAsToXContent(Metadata.builder().put(project).put(reservedStateMetadata).put(reservedStateMetadata1).build()) .toXContent( builder, new ToXContent.MapParams(Map.of("binary", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY)) @@ -280,8 +280,7 @@ public void testToXContentGateway_MultiProject() throws IOException { }, "index-graveyard" : { "tombstones" : [ ] - }, - "reserved_state" : { } + } } ], "reserved_state" : { } From d96df2970cefbb029bac9c17e2f1656056d26a4f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 1 Aug 2025 17:22:27 +0000 Subject: [PATCH 06/16] [CI] Auto commit changes from spotless --- .../cluster/metadata/Metadata.java | 3 ++- .../cluster/ClusterStateTests.java | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 458b305b218d5..8571f938b3e18 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -825,7 +825,8 @@ private Iterator toXContentChunkedWithSingleProjectFormat( } // make order deterministic - Iterator reservedStateMetadataIterator = reservedStateMetadata.entrySet().stream() + Iterator reservedStateMetadataIterator = reservedStateMetadata.entrySet() + .stream() .sorted(Map.Entry.comparingByKey()) .map(Map.Entry::getValue) .iterator(); diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index a5e597f5bc94a..dfd86d1f4fe0d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -91,8 +91,6 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -2255,7 +2253,10 @@ public void testSerialization() throws IOException { // check it deserializes ok NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); - ClusterState deserialisedClusterState = ClusterState.readFrom(new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry), null); + ClusterState deserialisedClusterState = ClusterState.readFrom( + new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry), + null + ); // check it matches the original object Metadata deserializedMetadata = deserialisedClusterState.metadata(); @@ -2271,9 +2272,14 @@ public void testCombinedReservedMetadataSerialization() throws IOException { .putCustom( ProjectStateRegistry.TYPE, ProjectStateRegistry.builder() - .putReservedStateMetadata(ProjectId.DEFAULT, ReservedStateMetadata.builder("file_settings").putHandler( - new ReservedStateHandlerMetadata("settings", Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey())) - ).build()) + .putReservedStateMetadata( + ProjectId.DEFAULT, + ReservedStateMetadata.builder("file_settings") + .putHandler( + new ReservedStateHandlerMetadata("settings", Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey())) + ) + .build() + ) .build() ) .build(); From 9045018802c293308aed9efdce4cedc8f26b7cda Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 1 Aug 2025 22:11:02 +0100 Subject: [PATCH 07/16] Fix tests --- .../cluster/project/ProjectStateRegistry.java | 14 ++++++++++++-- .../elasticsearch/cluster/ClusterStateTests.java | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index c84267f1fc626..f3575a4e695f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -29,8 +29,10 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -365,6 +367,7 @@ public ProjectStateRegistry build() { private record Entry(Settings settings, ImmutableOpenMap reservedStateMetadata) implements + ToXContentFragment, Writeable, Diffable { @@ -410,11 +413,18 @@ public void writeTo(StreamOutput out) throws IOException { } } - public void toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject("settings"); settings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); builder.endObject(); - ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator()); + + builder.startObject("reserved_state"); + for (ReservedStateMetadata reservedStateMetadata : reservedStateMetadata.values()) { + reservedStateMetadata.toXContent(builder, params); + } + builder.endObject(); + return builder; } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index dfd86d1f4fe0d..4424fe975c78c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -814,6 +814,17 @@ public void testToXContentWithMultipleProjects() throws IOException { "project.setting": "42", "project.setting2": "43" }, + "reserved_state": { + "file_settings": { + "handlers": { + "settings": { + "keys": ["project.setting", "project.setting2"] + } + }, + "version": 42, + "errors": null + } + }, "marked_for_deletion": true } ], @@ -933,6 +944,11 @@ private static ClusterState buildMultiProjectClusterState(DiscoveryNode... nodes projectId1, Settings.builder().put(PROJECT_SETTING.getKey(), 42).put(PROJECT_SETTING2.getKey(), 43).build() ) + .putReservedStateMetadata(projectId1, ReservedStateMetadata.builder("file_settings") + .putHandler(new ReservedStateHandlerMetadata("settings", + Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey()))) + .version(42L) + .build()) .markProjectForDeletion(projectId1) .build() ) From 1a497262d36590af78008c79d4e07edb723b0b31 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 1 Aug 2025 21:24:04 +0000 Subject: [PATCH 08/16] [CI] Auto commit changes from spotless --- .../cluster/project/ProjectStateRegistry.java | 2 -- .../elasticsearch/cluster/ClusterStateTests.java | 14 +++++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index f3575a4e695f2..5ce09f7398d28 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -29,8 +29,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 4424fe975c78c..b19c6918244e9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -944,11 +944,15 @@ private static ClusterState buildMultiProjectClusterState(DiscoveryNode... nodes projectId1, Settings.builder().put(PROJECT_SETTING.getKey(), 42).put(PROJECT_SETTING2.getKey(), 43).build() ) - .putReservedStateMetadata(projectId1, ReservedStateMetadata.builder("file_settings") - .putHandler(new ReservedStateHandlerMetadata("settings", - Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey()))) - .version(42L) - .build()) + .putReservedStateMetadata( + projectId1, + ReservedStateMetadata.builder("file_settings") + .putHandler( + new ReservedStateHandlerMetadata("settings", Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey())) + ) + .version(42L) + .build() + ) .markProjectForDeletion(projectId1) .build() ) From 4156e8f3341aa792fcf23102585344ec76412f94 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Mon, 4 Aug 2025 16:18:11 +0100 Subject: [PATCH 09/16] Address some review comments --- .../elasticsearch/cluster/ClusterState.java | 35 +-------------- .../cluster/metadata/Metadata.java | 42 ++--------------- .../ReservedProjectStateUpdateTask.java | 2 +- .../cluster/ClusterStateTests.java | 45 ------------------- 4 files changed, 7 insertions(+), 117 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 9defaa01a639c..9d019e74010e3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -24,10 +24,8 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; -import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.GlobalRoutingTable; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingNodes; @@ -75,7 +73,6 @@ import java.util.function.Consumer; import java.util.function.Function; -import static org.elasticsearch.cluster.metadata.Metadata.DEFAULT_PROJECT_ID; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; /** @@ -1319,13 +1316,7 @@ public void writeTo(StreamOutput out) throws IOException { clusterName.writeTo(out); out.writeLong(version); out.writeString(stateUUID); - if (out.getTransportVersion().before(TransportVersions.MULTI_PROJECT)) { - Map singleProjectReservedState = ProjectStateRegistry.get(this) - .reservedStateMetadata(Metadata.DEFAULT_PROJECT_ID); - metadata.writeTo(out, singleProjectReservedState); - } else { - metadata.writeTo(out); - } + metadata.writeTo(out); if (out.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT)) { routingTable.writeTo(out); } else { @@ -1375,33 +1366,11 @@ private static class ClusterStateDiff implements Diff { ); features = after.clusterFeatures.diff(before.clusterFeatures); - metadata = getMetadataDiff(before, after); + metadata = after.metadata.diff(before.metadata); blocks = after.blocks.diff(before.blocks); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); } - @FixForMultiProject - private Diff getMetadataDiff(ClusterState before, ClusterState after) { - final Diff metadata; - ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(after); - if (projectStateRegistry.size() == 1 && projectStateRegistry.hasProject(Metadata.DEFAULT_PROJECT_ID)) { - Map reservedStateMetadataBefore = ProjectStateRegistry.get(before) - .reservedStateMetadata(DEFAULT_PROJECT_ID); - Map reservedStateMetadataAfter = projectStateRegistry.reservedStateMetadata( - DEFAULT_PROJECT_ID - ); - DiffableUtils.MapDiff> diff = DiffableUtils.diff( - reservedStateMetadataBefore, - reservedStateMetadataAfter, - DiffableUtils.getStringKeySerializer() - ); - metadata = after.metadata.diff(before.metadata, diff); - } else { - metadata = after.metadata.diff(before.metadata); - } - return metadata; - } - ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { clusterName = new ClusterName(in); fromUuid = in.readString(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 8571f938b3e18..0d5cd3ee137b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -706,14 +706,7 @@ private static boolean projectMetadataEqual( @Override public Diff diff(Metadata previousState) { - return diff(previousState, DiffableUtils.emptyDiff()); - } - - public Diff diff( - Metadata previousState, - MapDiff> singleProjectReservedStateMetadata - ) { - return new MetadataDiff(previousState, this, singleProjectReservedStateMetadata); + return new MetadataDiff(previousState, this); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -852,10 +845,6 @@ private static class MetadataDiff implements Diff { private final Settings persistentSettings; private final Diff hashesOfConsistentSettings; private final ProjectMetadata.ProjectMetadataDiff singleProject; - private final DiffableUtils.MapDiff< - String, - ReservedStateMetadata, - Map> singleProjectReservedStateMetadata; private final MapDiff> multiProject; private final MapDiff> clusterCustoms; @@ -875,8 +864,7 @@ private static class MetadataDiff implements Diff { MetadataDiff( Metadata before, - Metadata after, - MapDiff> singleProjectReservedState + Metadata after ) { this.empty = before == after; this.fromNodeBeforeMultiProjectsSupport = false; // diff on this node, always after multi-projects, even when disabled @@ -889,11 +877,9 @@ private static class MetadataDiff implements Diff { if (before.isSingleProject() && after.isSingleProject()) { // single-project, just handle the project metadata diff itself singleProject = after.getSingleProject().diff(before.getSingleProject()); - singleProjectReservedStateMetadata = singleProjectReservedState; multiProject = null; } else { singleProject = null; - singleProjectReservedStateMetadata = null; multiProject = DiffableUtils.diff(before.projectMetadata, after.projectMetadata, ProjectId.PROJECT_ID_SERIALIZER); } @@ -1000,7 +986,6 @@ private MetadataDiff(StreamInput in) throws IOException { ); singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms); - singleProjectReservedStateMetadata = DiffableUtils.emptyDiff(); multiProject = null; } else { fromNodeBeforeMultiProjectsSupport = false; @@ -1017,7 +1002,6 @@ private MetadataDiff(StreamInput in) throws IOException { ); singleProject = null; - singleProjectReservedStateMetadata = null; multiProject = DiffableUtils.readJdkMapDiff( in, ProjectId.PROJECT_ID_SERIALIZER, @@ -1068,7 +1052,7 @@ public void writeTo(StreamOutput out) throws IOException { singleProject.indices().writeTo(out); singleProject.templates().writeTo(out); buildUnifiedCustomDiff().writeTo(out); - buildUnifiedReservedStateMetadataDiff().writeTo(out); + reservedStateMetadata.writeTo(out); } else { clusterCustoms.writeTo(out); reservedStateMetadata.writeTo(out); @@ -1114,15 +1098,6 @@ public void writeTo(StreamOutput out) throws IOException { } } - private Diff> buildUnifiedReservedStateMetadataDiff() { - return DiffableUtils.merge( - reservedStateMetadata, - singleProjectReservedStateMetadata, - DiffableUtils.getStringKeySerializer(), - RESERVED_DIFF_VALUE_READER - ); - } - @Override public Metadata apply(Metadata part) { if (empty) { @@ -1288,10 +1263,6 @@ private static void readClusterCustoms(StreamInput in, Builder builder) throws I @Override public void writeTo(StreamOutput out) throws IOException { - writeTo(out, Collections.emptyMap()); - } - - public void writeTo(StreamOutput out, Map singleProjectReservedStateMetadata) throws IOException { out.writeLong(version); out.writeString(clusterUUID); out.writeBoolean(clusterUUIDCommitted); @@ -1328,12 +1299,7 @@ public void writeTo(StreamOutput out, Map singleP ); VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms); - List combinedMetadata = new ArrayList<>( - reservedStateMetadata.size() + singleProjectReservedStateMetadata.size() - ); - combinedMetadata.addAll(reservedStateMetadata.values()); - combinedMetadata.addAll(singleProjectReservedStateMetadata.values()); - out.writeCollection(combinedMetadata); + out.writeCollection(reservedStateMetadata.values()); } else { VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java index a1633009ffde1..1b3e2661480f4 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/ReservedProjectStateUpdateTask.java @@ -82,7 +82,7 @@ protected ClusterState execute(ClusterState currentState) { ProjectStateRegistry.TYPE, ProjectStateRegistry.builder(updatedProjectStateRegistry).putReservedStateMetadata(projectId, result.v2()).build() ) - .putProjectMetadata(ProjectMetadata.builder(updatedProjectMetadata)) + .putProjectMetadata(updatedProjectMetadata) .build(); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index b19c6918244e9..3d8461bc4d29d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -2286,49 +2286,4 @@ public void testSerialization() throws IOException { assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).templates(), hasKey("template")); assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).indices(), hasKey("index")); } - - public void testCombinedReservedMetadataSerialization() throws IOException { - ClusterState clusterState = ClusterState.builder(buildClusterState()) - .putCustom( - ProjectStateRegistry.TYPE, - ProjectStateRegistry.builder() - .putReservedStateMetadata( - ProjectId.DEFAULT, - ReservedStateMetadata.builder("file_settings") - .putHandler( - new ReservedStateHandlerMetadata("settings", Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey())) - ) - .build() - ) - .build() - ) - .build(); - - TransportVersion oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT); - - BytesStreamOutput out = new BytesStreamOutput(); - out.setTransportVersion(oldVersion); - clusterState.writeTo(out); - - // check it deserializes ok - NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); - NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry); - in.setTransportVersion(oldVersion); - ClusterState deserialisedClusterState = ClusterState.readFrom(in, null); - - // check it matches the original object - Metadata deserializedMetadata = deserialisedClusterState.metadata(); - assertThat(deserializedMetadata.projects(), aMapWithSize(1)); - assertThat(deserializedMetadata.projects(), hasKey(ProjectId.DEFAULT)); - - assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).templates(), hasKey("template")); - assertThat(deserializedMetadata.getProject(ProjectId.DEFAULT).indices(), hasKey("index")); - - assertThat(deserializedMetadata.reservedStateMetadata(), hasKey("file_settings")); - ReservedStateMetadata fileSettings = deserializedMetadata.reservedStateMetadata().get("file_settings"); - assertThat(fileSettings.handlers(), aMapWithSize(1)); - assertThat(fileSettings.handlers(), hasKey("settings")); - ReservedStateHandlerMetadata settingsHandlerMetadata = fileSettings.handlers().get("settings"); - assertThat(settingsHandlerMetadata.keys(), equalTo(Set.of(PROJECT_SETTING.getKey(), PROJECT_SETTING2.getKey()))); - } } From e3911dddedf5d6be0213749bcd3f5a7a8df81545 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 4 Aug 2025 15:27:04 +0000 Subject: [PATCH 10/16] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/cluster/metadata/Metadata.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 0d5cd3ee137b2..a8372cd07be25 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -862,10 +862,7 @@ private static class MetadataDiff implements Diff { // This is used only when the node has a single project and needs to send the diff to an old node (wire BWC). private final MapDiff> combinedTasksDiff; - MetadataDiff( - Metadata before, - Metadata after - ) { + MetadataDiff(Metadata before, Metadata after) { this.empty = before == after; this.fromNodeBeforeMultiProjectsSupport = false; // diff on this node, always after multi-projects, even when disabled clusterUUID = after.clusterUUID; From a869c06881ac94d4532e74bdf045b629a821c823 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Thu, 21 Aug 2025 18:50:19 +0100 Subject: [PATCH 11/16] Address review comments --- .../java/org/elasticsearch/cluster/ClusterState.java | 1 - .../java/org/elasticsearch/cluster/DiffableUtils.java | 10 +++++----- .../cluster/metadata/ProjectMetadata.java | 4 ++++ .../cluster/project/ProjectStateRegistry.java | 10 +++++----- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 9d019e74010e3..d5356bd54b845 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -1365,7 +1365,6 @@ private static class ClusterStateDiff implements Diff { COMPATIBILITY_VERSIONS_VALUE_SERIALIZER ); features = after.clusterFeatures.diff(before.clusterFeatures); - metadata = after.metadata.diff(before.metadata); blocks = after.blocks.diff(before.blocks); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); diff --git a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java index aff5de0b1737d..542f6879ffd41 100644 --- a/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/elasticsearch/cluster/DiffableUtils.java @@ -104,8 +104,8 @@ public static > MapDiff emptyDiff() { */ @SuppressWarnings("unchecked") public static , T1 extends T, T2 extends T, M extends Map> MapDiff merge( - MapDiff> diff1, - MapDiff> diff2, + MapDiff> diff1, + MapDiff> diff2, KeySerializer keySerializer ) { return merge(diff1, diff2, keySerializer, DiffableValueSerializer.getWriteOnlyInstance()); @@ -116,8 +116,8 @@ public static , T1 extends T, T2 extends T, M extends M */ @SuppressWarnings("unchecked") public static > MapDiff merge( - MapDiff> diff1, - MapDiff> diff2, + MapDiff> diff1, + MapDiff> diff2, KeySerializer keySerializer, ValueSerializer valueSerializer ) { @@ -130,7 +130,7 @@ public static > MapDiff (T) val), mapEntries(diff2.getUpserts(), val -> (T) val) ).toList(); - return new MapDiff<>(keySerializer, valueSerializer, deletes, diffs, upserts, DiffableUtils::createImmutableMapBuilder); + return new MapDiff(keySerializer, valueSerializer, deletes, diffs, upserts, DiffableUtils::createImmutableMapBuilder); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 5f7df4daed4bd..cb3a39b24fc38 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -2058,6 +2059,7 @@ static boolean assertDataStreams(Map indices, DataStreamM return true; } + @FixForMultiProject(description = "Remove reading reserved_state and settings") public static ProjectMetadata fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.currentToken(); if (token == null) { @@ -2078,6 +2080,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep } } else if (token == XContentParser.Token.START_OBJECT) { switch (currentFieldName) { + // Remove this case "reserved_state" -> { while (parser.nextToken() != XContentParser.Token.END_OBJECT) { ReservedStateMetadata.fromXContent(parser); @@ -2093,6 +2096,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep projectBuilder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); } } + // Remove this case "settings" -> { Settings.fromXContent(parser); } diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index 5ce09f7398d28..6bb7dfc627295 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -316,7 +316,7 @@ private Builder(ProjectStateRegistry original) { this.projectsMarkedForDeletionGeneration = original.projectsMarkedForDeletionGeneration; } - private void modifyEntry(ProjectId projectId, Function modifier) { + private void updateEntry(ProjectId projectId, Function modifier) { Entry entry = projectsEntries.get(projectId); if (entry == null) { entry = new Entry(); @@ -326,12 +326,12 @@ private void modifyEntry(ProjectId projectId, Function modifier) { } public Builder putProjectSettings(ProjectId projectId, Settings settings) { - modifyEntry(projectId, entry -> entry.withSettings(settings)); + updateEntry(projectId, entry -> entry.withSettings(settings)); return this; } public Builder putReservedStateMetadata(ProjectId projectId, ReservedStateMetadata reservedStateMetadata) { - modifyEntry(projectId, entry -> entry.withReservedStateMetadata(reservedStateMetadata)); + updateEntry(projectId, entry -> entry.withReservedStateMetadata(reservedStateMetadata)); return this; } @@ -397,10 +397,10 @@ public Entry withSettings(Settings settings) { } public Entry withReservedStateMetadata(ReservedStateMetadata reservedStateMetadata) { - ImmutableOpenMap build = ImmutableOpenMap.builder(this.reservedStateMetadata) + ImmutableOpenMap reservedStateMetadataMap = ImmutableOpenMap.builder(this.reservedStateMetadata) .fPut(reservedStateMetadata.namespace(), reservedStateMetadata) .build(); - return new Entry(settings, build); + return new Entry(settings, reservedStateMetadataMap); } @Override From 7c54ef34756720ca3268ed8cdcbbec2dd3f91c42 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 22 Aug 2025 15:51:43 +0100 Subject: [PATCH 12/16] Fix projects state registry filtering in GetClusterState API --- .../state/TransportClusterStateAction.java | 9 ++++++- .../TransportClusterStateActionTests.java | 25 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index cfedcc4bd7267..916b1c2a58e0c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.routing.GlobalRoutingTable; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.service.ClusterService; @@ -162,13 +163,19 @@ private ClusterState filterClusterState(final ClusterState inputState) { } final Metadata.Builder mdBuilder = Metadata.builder(inputState.metadata()); final GlobalRoutingTable.Builder rtBuilder = GlobalRoutingTable.builder(inputState.globalRoutingTable()); + final ProjectStateRegistry.Builder psBuilder = ProjectStateRegistry.builder(inputState); for (var projectId : metadata.projects().keySet()) { if (projectIds.contains(projectId) == false) { mdBuilder.removeProject(projectId); rtBuilder.removeProject(projectId); + psBuilder.removeProject(projectId); } } - return ClusterState.builder(inputState).metadata(mdBuilder.build()).routingTable(rtBuilder.build()).build(); + return ClusterState.builder(inputState) + .metadata(mdBuilder.build()) + .routingTable(rtBuilder.build()) + .putCustom(ProjectStateRegistry.TYPE, psBuilder.build()) + .build(); } @SuppressForbidden(reason = "exposing ClusterState#compatibilityVersions requires reading them") diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java index 0ef26813eb6c0..8bd27d148873f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.cluster.project.DefaultProjectResolver; import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.project.ProjectStateRegistry; import org.elasticsearch.cluster.project.TestProjectResolvers; import org.elasticsearch.cluster.routing.GlobalRoutingTableTestHelper; import org.elasticsearch.cluster.routing.RoutingTable; @@ -169,6 +170,13 @@ public void testGetClusterStateForManyProjects() throws Exception { assertThat(routingTable, notNullValue()); assertThat(routingTable.indicesRouting().keySet(), containsInAnyOrder(expectedIndices)); } + if (request.customs()) { + ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(response.getState()); + assertThat(projectStateRegistry.size(), equalTo(numberOfProjects)); + Settings projectSettings = projectStateRegistry.getProjectSettings(projectId); + assertThat(projectSettings, notNullValue()); + assertThat(projectSettings.keySet(), contains("setting_1")); + } } } @@ -194,6 +202,13 @@ private static void assertSingleProjectResponse( } else { assertThat(routingTables.get(projectId).indicesRouting(), anEmptyMap()); } + if (request.customs()) { + ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(response.getState()); + assertThat(projectStateRegistry.size(), equalTo(1)); + Settings projectSettings = projectStateRegistry.getProjectSettings(projectId); + assertThat(projectSettings, notNullValue()); + assertThat(projectSettings.keySet(), contains("setting_1")); + } } private ClusterStateResponse executeAction(ProjectResolver projectResolver, ClusterStateRequest request, ClusterState state) @@ -232,7 +247,7 @@ private static ClusterStateRequest buildRandomRequest(Set indexNames) { request.nodes(randomBoolean()); request.routingTable(randomBoolean()); request.blocks(randomBoolean()); - request.customs(randomBoolean()); + request.customs(true); return request; } @@ -241,9 +256,15 @@ private static ClusterState buildClusterState(ProjectMetadata.Builder... project Arrays.stream(projects).forEach(metadataBuilder::put); final var metadata = metadataBuilder.build(); - return ClusterState.builder(new ClusterName(randomAlphaOfLengthBetween(4, 12))) + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName(randomAlphaOfLengthBetween(4, 12))); + ProjectStateRegistry.Builder psBuilder = ProjectStateRegistry.builder(); + for (ProjectMetadata.Builder project : projects) { + psBuilder.putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build()); + } + return csBuilder .metadata(metadata) .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew)) + .putCustom(ProjectStateRegistry.TYPE, psBuilder.build()) .build(); } From 28b46ce3278b320bd0574aecc5cabdee1c52b2b2 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 22 Aug 2025 15:54:45 +0100 Subject: [PATCH 13/16] Update docs/changelog/133401.yaml --- docs/changelog/133401.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/133401.yaml diff --git a/docs/changelog/133401.yaml b/docs/changelog/133401.yaml new file mode 100644 index 0000000000000..a778adc826f7c --- /dev/null +++ b/docs/changelog/133401.yaml @@ -0,0 +1,5 @@ +pr: 133401 +summary: Fix projects state registry filtering in `GetClusterState` API +area: Infra/Core +type: bug +issues: [] From f33aa531d8e65cbe62714ef2d8375f7f464f8ac4 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Fri, 22 Aug 2025 15:55:47 +0100 Subject: [PATCH 14/16] Delete docs/changelog/133401.yaml --- docs/changelog/133401.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/133401.yaml diff --git a/docs/changelog/133401.yaml b/docs/changelog/133401.yaml deleted file mode 100644 index a778adc826f7c..0000000000000 --- a/docs/changelog/133401.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 133401 -summary: Fix projects state registry filtering in `GetClusterState` API -area: Infra/Core -type: bug -issues: [] From 815217311a24e6392ba100242035972e00a02ac9 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 22 Aug 2025 15:03:56 +0000 Subject: [PATCH 15/16] [CI] Auto commit changes from spotless --- .../admin/cluster/state/TransportClusterStateActionTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java index 8bd27d148873f..1fd3c15f1a18f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java @@ -261,8 +261,7 @@ private static ClusterState buildClusterState(ProjectMetadata.Builder... project for (ProjectMetadata.Builder project : projects) { psBuilder.putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build()); } - return csBuilder - .metadata(metadata) + return csBuilder.metadata(metadata) .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew)) .putCustom(ProjectStateRegistry.TYPE, psBuilder.build()) .build(); From b473065b3c63d48b4fcbd687ee616d49c9f54468 Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Thu, 28 Aug 2025 15:07:48 +0100 Subject: [PATCH 16/16] Address minor review comments --- .../org/elasticsearch/cluster/metadata/ProjectMetadata.java | 6 +++--- .../elasticsearch/cluster/project/ProjectStateRegistry.java | 4 ++-- .../reservedstate/service/FileSettingsService.java | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index cb3a39b24fc38..a5995dd15490c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -2059,7 +2059,7 @@ static boolean assertDataStreams(Map indices, DataStreamM return true; } - @FixForMultiProject(description = "Remove reading reserved_state and settings") + @FixForMultiProject(description = "Remove reading reserved_state and settings") // ES-12795 public static ProjectMetadata fromXContent(XContentParser parser) throws IOException { XContentParser.Token token = parser.currentToken(); if (token == null) { @@ -2080,7 +2080,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep } } else if (token == XContentParser.Token.START_OBJECT) { switch (currentFieldName) { - // Remove this + // Remove this (ES-12795) case "reserved_state" -> { while (parser.nextToken() != XContentParser.Token.END_OBJECT) { ReservedStateMetadata.fromXContent(parser); @@ -2096,7 +2096,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep projectBuilder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); } } - // Remove this + // Remove this (ES-12795) case "settings" -> { Settings.fromXContent(parser); } diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index 6bb7dfc627295..dd85bd0e68c70 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -40,7 +40,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; /** @@ -316,7 +316,7 @@ private Builder(ProjectStateRegistry original) { this.projectsMarkedForDeletionGeneration = original.projectsMarkedForDeletionGeneration; } - private void updateEntry(ProjectId projectId, Function modifier) { + private void updateEntry(ProjectId projectId, UnaryOperator modifier) { Entry entry = projectsEntries.get(projectId); if (entry == null) { entry = new Entry(); diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java index 2eaf50ca3849f..47182c30dd30b 100644 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java +++ b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.FixForMultiProject; import org.elasticsearch.env.Environment; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorImpact; @@ -131,6 +132,7 @@ public Path watchedFile() { * @param mdBuilder the current metadata builder for the new cluster state * @param projectId the project associated with the restore */ + @FixForMultiProject(description = "Simplify parameters (ES-12796)") public void handleSnapshotRestore( ClusterState clusterState, ClusterState.Builder builder,