Move per-project settings out of ProjectMetadata (elastic#129068)

To better support project restoration after deletion, this change moves project Settings from ProjectMetadata to the new custom in the ClusterState. It also introduces a new transport version for cluster state serialization. Reserved cluster state for project settings remains within ProjectMetadata.

Note: In mixed-version multiproject clusters, this may cause existing settings for projects to temporarily disappear until all nodes have been upgraded and restarted.

Author: alexey-ivanov-es

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -314,6 +314,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE = def(9_105_0_00);
     public static final TransportVersion ML_INFERENCE_SAGEMAKER_ELASTIC = def(9_106_0_00);
     public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS = def(9_107_0_00);
+    public static final TransportVersion CLUSTER_STATE_PROJECTS_SETTINGS = def(9_108_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.metadata.StreamsMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
+import org.elasticsearch.cluster.project.ProjectStateRegistry;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy;
@@ -292,6 +293,7 @@ public static List<Entry> getNamedWriteables() {
             RegisteredPolicySnapshots::new,
             RegisteredPolicySnapshots.RegisteredSnapshotsDiff::new
         );
+        registerClusterCustom(entries, ProjectStateRegistry.TYPE, ProjectStateRegistry::new, ProjectStateRegistry::readDiffFrom);
         // Secrets
         registerClusterCustom(entries, ClusterSecrets.TYPE, ClusterSecrets::new, ClusterSecrets::readDiffFrom);
         registerProjectCustom(entries, ProjectSecrets.TYPE, ProjectSecrets::new, ProjectSecrets::readDiffFrom);

server/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -1047,7 +1047,7 @@ public Builder(ClusterState state) {
         public Builder(ClusterName clusterName) {
             this.compatibilityVersions = new HashMap<>();
             this.nodeFeatures = new HashMap<>();
-            customs = ImmutableOpenMap.builder();
+            this.customs = ImmutableOpenMap.builder();
             this.clusterName = clusterName;
         }
 
@@ -1330,7 +1330,6 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     private static class ClusterStateDiff implements Diff<ClusterState> {
-
         private final long toVersion;
 
         private final String fromUuid;

server/src/main/java/org/elasticsearch/cluster/ProjectState.java

@@ -12,8 +12,10 @@
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.project.ProjectStateRegistry;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 
 import java.util.Objects;
 import java.util.function.Consumer;
@@ -26,6 +28,7 @@ public final class ProjectState {
     private final ClusterState cluster;
     private final ProjectId project;
     private final ProjectMetadata projectMetadata;
+    private final Settings projectSettings;
     private final RoutingTable routingTable;
 
     ProjectState(ClusterState clusterState, ProjectId projectId) {
@@ -34,6 +37,7 @@ public final class ProjectState {
         this.cluster = clusterState;
         this.project = projectId;
         this.projectMetadata = clusterState.metadata().getProject(projectId);
+        this.projectSettings = ProjectStateRegistry.getProjectSettings(projectId, clusterState);
         this.routingTable = clusterState.routingTable(projectId);
     }
 
@@ -57,6 +61,10 @@ public ClusterBlocks blocks() {
         return cluster().blocks();
     }
 
+    public Settings settings() {
+        return projectSettings;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == this) return true;

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -985,13 +985,7 @@ private MetadataDiff(StreamInput in) throws IOException {
                     RESERVED_DIFF_VALUE_READER
                 );
 
-                singleProject = new ProjectMetadata.ProjectMetadataDiff(
-                    indices,
-                    templates,
-                    projectCustoms,
-                    DiffableUtils.emptyDiff(),
-                    Settings.EMPTY_DIFF
-                );
+                singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms, DiffableUtils.emptyDiff());
                 multiProject = null;
             } else {
                 fromNodeBeforeMultiProjectsSupport = false;

server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java

@@ -106,8 +106,6 @@ public class ProjectMetadata implements Iterable<IndexMetadata>, Diffable<Projec
     private volatile SortedMap<String, IndexAbstraction> indicesLookup;
     private final Map<String, MappingMetadata> mappingsByHash;
 
-    private final Settings settings;
-
     private final IndexVersion oldestIndexVersion;
 
     public static final ClusterBlock PROJECT_UNDER_DELETION_BLOCK = new ClusterBlock(
@@ -138,7 +136,6 @@ private ProjectMetadata(
         String[] visibleClosedIndices,
         SortedMap<String, IndexAbstraction> indicesLookup,
         Map<String, MappingMetadata> mappingsByHash,
-        Settings settings,
         IndexVersion oldestIndexVersion
     ) {
         this.id = id;
@@ -157,7 +154,6 @@ private ProjectMetadata(
         this.visibleClosedIndices = visibleClosedIndices;
         this.indicesLookup = indicesLookup;
         this.mappingsByHash = mappingsByHash;
-        this.settings = settings;
         this.oldestIndexVersion = oldestIndexVersion;
         assert assertConsistent();
     }
@@ -239,7 +235,6 @@ public ProjectMetadata withLifecycleState(Index index, LifecycleExecutionState l
             visibleClosedIndices,
             indicesLookup,
             mappingsByHash,
-            settings,
             oldestIndexVersion
         );
     }
@@ -273,7 +268,6 @@ public ProjectMetadata withIndexSettingsUpdates(Map<Index, Settings> updates) {
             visibleClosedIndices,
             indicesLookup,
             mappingsByHash,
-            settings,
             oldestIndexVersion
         );
     }
@@ -308,7 +302,6 @@ public ProjectMetadata withAllocationAndTermUpdatesOnly(Map<String, IndexMetadat
             visibleClosedIndices,
             indicesLookup,
             mappingsByHash,
-            settings,
             oldestIndexVersion
         );
     }
@@ -397,7 +390,6 @@ public ProjectMetadata withAddedIndex(IndexMetadata index) {
             updatedVisibleClosedIndices,
             null,
             updatedMappingsByHash,
-            settings,
             IndexVersion.min(index.getCompatibilityVersion(), oldestIndexVersion)
         );
     }
@@ -749,10 +741,6 @@ public Map<String, IndexTemplateMetadata> templates() {
         return templates;
     }
 
-    public Settings settings() {
-        return settings;
-    }
-
     /**
      * Checks whether the provided index is a data stream.
      */
@@ -1123,9 +1111,6 @@ public boolean isIndexManagedByILM(IndexMetadata indexMetadata) {
     }
 
     static boolean isStateEquals(ProjectMetadata project1, ProjectMetadata project2) {
-        if (project1.settings().equals(project2.settings()) == false) {
-            return false;
-        }
         if (project1.templates().equals(project2.templates()) == false) {
             return false;
         }
@@ -1159,7 +1144,6 @@ public static class Builder {
         private final ImmutableOpenMap.Builder<String, IndexTemplateMetadata> templates;
         private final ImmutableOpenMap.Builder<String, Metadata.ProjectCustom> customs;
         private final ImmutableOpenMap.Builder<String, ReservedStateMetadata> reservedStateMetadata;
-        private Settings settings = Settings.EMPTY;
 
         private SortedMap<String, IndexAbstraction> previousIndicesLookup;
 
@@ -1177,7 +1161,6 @@ public static class Builder {
             this.templates = ImmutableOpenMap.builder(projectMetadata.templates);
             this.customs = ImmutableOpenMap.builder(projectMetadata.customs);
             this.reservedStateMetadata = ImmutableOpenMap.builder(projectMetadata.reservedStateMetadata);
-            this.settings = projectMetadata.settings;
             this.previousIndicesLookup = projectMetadata.indicesLookup;
             this.mappingsByHash = new HashMap<>(projectMetadata.mappingsByHash);
             this.checkForUnusedMappings = false;
@@ -1551,11 +1534,6 @@ public Builder removeReservedState(ReservedStateMetadata metadata) {
             return this;
         }
 
-        public Builder settings(Settings settings) {
-            this.settings = settings;
-            return this;
-        }
-
         public Builder indexGraveyard(final IndexGraveyard indexGraveyard) {
             return putCustom(IndexGraveyard.TYPE, indexGraveyard);
         }
@@ -1715,7 +1693,6 @@ public ProjectMetadata build(boolean skipNameCollisionChecks) {
                 visibleClosedIndicesArray,
                 indicesLookup,
                 Collections.unmodifiableMap(mappingsByHash),
-                settings,
                 IndexVersion.fromId(oldestIndexVersionId)
             );
         }
@@ -2143,7 +2120,7 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep
                             }
                         }
                         case "settings" -> {
-                            projectBuilder.settings(Settings.fromXContent(parser));
+                            Settings.fromXContent(parser);
                         }
                         default -> Metadata.Builder.parseCustomObject(
                             parser,
@@ -2190,11 +2167,6 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params p) {
             ),
             indices,
             customs,
-            multiProject ? Iterators.single((builder, params) -> {
-                builder.startObject("settings");
-                settings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
-                return builder.endObject();
-            }) : Collections.emptyIterator(),
             multiProject
                 ? ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator())
                 : Collections.emptyIterator()
@@ -2229,8 +2201,9 @@ public static ProjectMetadata readFrom(StreamInput in) throws IOException {
             builder.put(ReservedStateMetadata.readFrom(in));
         }
 
-        if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
-            builder.settings(Settings.readSettingsFromStream(in));
+        if (in.getTransportVersion()
+            .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) {
+            Settings.readSettingsFromStream(in);
         }
 
         return builder.build();
@@ -2264,8 +2237,9 @@ public void writeTo(StreamOutput out) throws IOException {
         VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
         out.writeCollection(reservedStateMetadata.values());
 
-        if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
-            settings.writeTo(out);
+        if (out.getTransportVersion()
+            .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) {
+            Settings.EMPTY.writeTo(out);
         }
     }
 
@@ -2286,15 +2260,13 @@ static class ProjectMetadataDiff implements Diff<ProjectMetadata> {
             String,
             ReservedStateMetadata,
             ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata;
-        private final Diff<Settings> settingsDiff;
 
         private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) {
             if (before == after) {
                 indices = DiffableUtils.emptyDiff();
                 templates = DiffableUtils.emptyDiff();
                 customs = DiffableUtils.emptyDiff();
                 reservedStateMetadata = DiffableUtils.emptyDiff();
-                settingsDiff = Settings.EMPTY_DIFF;
             } else {
                 indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
                 templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
@@ -2309,22 +2281,19 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) {
                     after.reservedStateMetadata,
                     DiffableUtils.getStringKeySerializer()
                 );
-                settingsDiff = after.settings.diff(before.settings);
             }
         }
 
         ProjectMetadataDiff(
             DiffableUtils.MapDiff<String, IndexMetadata, ImmutableOpenMap<String, IndexMetadata>> indices,
             DiffableUtils.MapDiff<String, IndexTemplateMetadata, ImmutableOpenMap<String, IndexTemplateMetadata>> templates,
             DiffableUtils.MapDiff<String, Metadata.ProjectCustom, ImmutableOpenMap<String, Metadata.ProjectCustom>> customs,
-            DiffableUtils.MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata,
-            Diff<Settings> settingsDiff
+            DiffableUtils.MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata
         ) {
             this.indices = indices;
             this.templates = templates;
             this.customs = customs;
             this.reservedStateMetadata = reservedStateMetadata;
-            this.settingsDiff = settingsDiff;
         }
 
         ProjectMetadataDiff(StreamInput in) throws IOException {
@@ -2336,10 +2305,9 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) {
                 DiffableUtils.getStringKeySerializer(),
                 RESERVED_DIFF_VALUE_READER
             );
-            if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
-                settingsDiff = Settings.readSettingsDiffFromStream(in);
-            } else {
-                settingsDiff = Settings.EMPTY_DIFF;
+            if (in.getTransportVersion()
+                .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) {
+                Settings.readSettingsDiffFromStream(in);
             }
         }
 
@@ -2365,18 +2333,15 @@ public void writeTo(StreamOutput out) throws IOException {
             templates.writeTo(out);
             customs.writeTo(out);
             reservedStateMetadata.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
-                settingsDiff.writeTo(out);
+            if (out.getTransportVersion()
+                .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) {
+                Settings.EMPTY_DIFF.writeTo(out);
             }
         }
 
         @Override
         public ProjectMetadata apply(ProjectMetadata part) {
-            if (indices.isEmpty()
-                && templates.isEmpty()
-                && customs.isEmpty()
-                && reservedStateMetadata.isEmpty()
-                && settingsDiff == Settings.EMPTY_DIFF) {
+            if (indices.isEmpty() && templates.isEmpty() && customs.isEmpty() && reservedStateMetadata.isEmpty()) {
                 // nothing to do
                 return part;
             }
@@ -2391,7 +2356,6 @@ public ProjectMetadata apply(ProjectMetadata part) {
                 && builder.dataStreamMetadata() == part.custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY)) {
                 builder.previousIndicesLookup = part.indicesLookup;
             }
-            builder.settings = settingsDiff.apply(part.settings);
             return builder.build(true);
         }
     }