New per-project only settings can be defined and used by components (elastic#127280)

This change introduces Settings to ProjectMetadata and adds project scope support for Setting.

For now, project-scoped settings are independent from cluster settings and do not fall back to cluster-level settings.
Also, setting update consumers do not yet work correctly for project-scoped settings. These issues will be addressed separately in future PRs.

Author: alexey-ivanov-es

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java

@@ -71,7 +71,8 @@ public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor<G
         "ingest.geoip.downloader.enabled",
         ENABLED_DEFAULT,
         Setting.Property.Dynamic,
-        Setting.Property.NodeScope
+        Setting.Property.NodeScope,
+        Setting.Property.ProjectScope
     );
     public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting(
         "ingest.geoip.downloader.poll.interval",

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -229,6 +229,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER = def(9_063_0_00);
     public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00);
     public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00);
+    public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -950,7 +950,13 @@ private MetadataDiff(StreamInput in) throws IOException {
                     RESERVED_DIFF_VALUE_READER
                 );
 
-                singleProject = new ProjectMetadata.ProjectMetadataDiff(indices, templates, projectCustoms, DiffableUtils.emptyDiff());
+                singleProject = new ProjectMetadata.ProjectMetadataDiff(
+                    indices,
+                    templates,
+                    projectCustoms,
+                    DiffableUtils.emptyDiff(),
+                    Settings.EMPTY_DIFF
+                );
                 multiProject = null;
             } else {
                 fromNodeBeforeMultiProjectsSupport = false;

server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java

@@ -102,6 +102,8 @@ public class ProjectMetadata implements Iterable<IndexMetadata>, Diffable<Projec
     private volatile SortedMap<String, IndexAbstraction> indicesLookup;
     private final Map<String, MappingMetadata> mappingsByHash;
 
+    private final Settings settings;
+
     private final IndexVersion oldestIndexVersion;
 
     @SuppressWarnings("this-escape")
@@ -122,6 +124,7 @@ private ProjectMetadata(
         String[] visibleClosedIndices,
         SortedMap<String, IndexAbstraction> indicesLookup,
         Map<String, MappingMetadata> mappingsByHash,
+        Settings settings,
         IndexVersion oldestIndexVersion
     ) {
         this.id = id;
@@ -140,6 +143,7 @@ private ProjectMetadata(
         this.visibleClosedIndices = visibleClosedIndices;
         this.indicesLookup = indicesLookup;
         this.mappingsByHash = mappingsByHash;
+        this.settings = settings;
         this.oldestIndexVersion = oldestIndexVersion;
         assert assertConsistent();
     }
@@ -221,6 +225,7 @@ public ProjectMetadata withLifecycleState(Index index, LifecycleExecutionState l
             visibleClosedIndices,
             indicesLookup,
             mappingsByHash,
+            settings,
             oldestIndexVersion
         );
     }
@@ -254,6 +259,7 @@ public ProjectMetadata withIndexSettingsUpdates(Map<Index, Settings> updates) {
             visibleClosedIndices,
             indicesLookup,
             mappingsByHash,
+            settings,
             oldestIndexVersion
         );
     }
@@ -288,6 +294,7 @@ public ProjectMetadata withAllocationAndTermUpdatesOnly(Map<String, IndexMetadat
             visibleClosedIndices,
             indicesLookup,
             mappingsByHash,
+            settings,
             oldestIndexVersion
         );
     }
@@ -376,6 +383,7 @@ public ProjectMetadata withAddedIndex(IndexMetadata index) {
             updatedVisibleClosedIndices,
             null,
             updatedMappingsByHash,
+            settings,
             IndexVersion.min(index.getCompatibilityVersion(), oldestIndexVersion)
         );
     }
@@ -727,6 +735,10 @@ public Map<String, IndexTemplateMetadata> templates() {
         return templates;
     }
 
+    public Settings settings() {
+        return settings;
+    }
+
     /**
      * Checks whether the provided index is a data stream.
      */
@@ -1097,6 +1109,9 @@ public boolean isIndexManagedByILM(IndexMetadata indexMetadata) {
     }
 
     static boolean isStateEquals(ProjectMetadata project1, ProjectMetadata project2) {
+        if (project1.settings().equals(project2.settings()) == false) {
+            return false;
+        }
         if (project1.templates().equals(project2.templates()) == false) {
             return false;
         }
@@ -1130,6 +1145,7 @@ public static class Builder {
         private final ImmutableOpenMap.Builder<String, IndexTemplateMetadata> templates;
         private final ImmutableOpenMap.Builder<String, Metadata.ProjectCustom> customs;
         private final ImmutableOpenMap.Builder<String, ReservedStateMetadata> reservedStateMetadata;
+        private Settings settings = Settings.EMPTY;
 
         private SortedMap<String, IndexAbstraction> previousIndicesLookup;
 
@@ -1147,6 +1163,7 @@ public static class Builder {
             this.templates = ImmutableOpenMap.builder(projectMetadata.templates);
             this.customs = ImmutableOpenMap.builder(projectMetadata.customs);
             this.reservedStateMetadata = ImmutableOpenMap.builder(projectMetadata.reservedStateMetadata);
+            this.settings = projectMetadata.settings;
             this.previousIndicesLookup = projectMetadata.indicesLookup;
             this.mappingsByHash = new HashMap<>(projectMetadata.mappingsByHash);
             this.checkForUnusedMappings = false;
@@ -1525,6 +1542,11 @@ public Builder removeReservedState(ReservedStateMetadata metadata) {
             return this;
         }
 
+        public Builder settings(Settings settings) {
+            this.settings = settings;
+            return this;
+        }
+
         public Builder indexGraveyard(final IndexGraveyard indexGraveyard) {
             return putCustom(IndexGraveyard.TYPE, indexGraveyard);
         }
@@ -1684,6 +1706,7 @@ public ProjectMetadata build(boolean skipNameCollisionChecks) {
                 visibleClosedIndicesArray,
                 indicesLookup,
                 Collections.unmodifiableMap(mappingsByHash),
+                settings,
                 IndexVersion.fromId(oldestIndexVersionId)
             );
         }
@@ -2110,6 +2133,9 @@ public static ProjectMetadata fromXContent(XContentParser parser) throws IOExcep
                                 projectBuilder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
                             }
                         }
+                        case "settings" -> {
+                            projectBuilder.settings(Settings.fromXContent(parser));
+                        }
                         default -> Metadata.Builder.parseCustomObject(
                             parser,
                             currentFieldName,
@@ -2155,6 +2181,11 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params p) {
             ),
             indices,
             customs,
+            multiProject ? Iterators.single((builder, params) -> {
+                builder.startObject("settings");
+                settings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+                return builder.endObject();
+            }) : Collections.emptyIterator(),
             multiProject
                 ? ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator())
                 : Collections.emptyIterator()
@@ -2189,6 +2220,10 @@ public static ProjectMetadata readFrom(StreamInput in) throws IOException {
             builder.put(ReservedStateMetadata.readFrom(in));
         }
 
+        if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
+            builder.settings(Settings.readSettingsFromStream(in));
+        }
+
         return builder.build();
     }
 
@@ -2229,6 +2264,10 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         VersionedNamedWriteable.writeVersionedWriteables(out, filteredCustoms);
         out.writeCollection(reservedStateMetadata.values());
+
+        if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
+            settings.writeTo(out);
+        }
     }
 
     // this needs to be package accessible for bwc serialization in Metadata.java
@@ -2248,13 +2287,15 @@ static class ProjectMetadataDiff implements Diff<ProjectMetadata> {
             String,
             ReservedStateMetadata,
             ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata;
+        private final Diff<Settings> settingsDiff;
 
         private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) {
             if (before == after) {
                 indices = DiffableUtils.emptyDiff();
                 templates = DiffableUtils.emptyDiff();
                 customs = DiffableUtils.emptyDiff();
                 reservedStateMetadata = DiffableUtils.emptyDiff();
+                settingsDiff = Settings.EMPTY_DIFF;
             } else {
                 indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
                 templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
@@ -2269,19 +2310,22 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) {
                     after.reservedStateMetadata,
                     DiffableUtils.getStringKeySerializer()
                 );
+                settingsDiff = after.settings.diff(before.settings);
             }
         }
 
         ProjectMetadataDiff(
             DiffableUtils.MapDiff<String, IndexMetadata, ImmutableOpenMap<String, IndexMetadata>> indices,
             DiffableUtils.MapDiff<String, IndexTemplateMetadata, ImmutableOpenMap<String, IndexTemplateMetadata>> templates,
             DiffableUtils.MapDiff<String, Metadata.ProjectCustom, ImmutableOpenMap<String, Metadata.ProjectCustom>> customs,
-            DiffableUtils.MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata
+            DiffableUtils.MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata,
+            Diff<Settings> settingsDiff
         ) {
             this.indices = indices;
             this.templates = templates;
             this.customs = customs;
             this.reservedStateMetadata = reservedStateMetadata;
+            this.settingsDiff = settingsDiff;
         }
 
         ProjectMetadataDiff(StreamInput in) throws IOException {
@@ -2293,6 +2337,11 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) {
                 DiffableUtils.getStringKeySerializer(),
                 RESERVED_DIFF_VALUE_READER
             );
+            if (in.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
+                settingsDiff = Settings.readSettingsDiffFromStream(in);
+            } else {
+                settingsDiff = Settings.EMPTY_DIFF;
+            }
         }
 
         Diff<ImmutableOpenMap<String, IndexMetadata>> indices() {
@@ -2317,11 +2366,18 @@ public void writeTo(StreamOutput out) throws IOException {
             templates.writeTo(out);
             customs.writeTo(out);
             reservedStateMetadata.writeTo(out);
+            if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
+                settingsDiff.writeTo(out);
+            }
         }
 
         @Override
         public ProjectMetadata apply(ProjectMetadata part) {
-            if (indices.isEmpty() && templates.isEmpty() && customs.isEmpty() && reservedStateMetadata.isEmpty()) {
+            if (indices.isEmpty()
+                && templates.isEmpty()
+                && customs.isEmpty()
+                && reservedStateMetadata.isEmpty()
+                && settingsDiff == Settings.EMPTY_DIFF) {
                 // nothing to do
                 return part;
             }
@@ -2336,13 +2392,14 @@ public ProjectMetadata apply(ProjectMetadata part) {
                 && builder.dataStreamMetadata() == part.custom(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY)) {
                 builder.previousIndicesLookup = part.indicesLookup;
             }
+            builder.settings = settingsDiff.apply(part.settings);
             return builder.build(true);
         }
 
         ProjectMetadataDiff withCustoms(
             DiffableUtils.MapDiff<String, Metadata.ProjectCustom, ImmutableOpenMap<String, Metadata.ProjectCustom>> customs
         ) {
-            return new ProjectMetadataDiff(indices, templates, customs, reservedStateMetadata);
+            return new ProjectMetadataDiff(indices, templates, customs, reservedStateMetadata, settingsDiff);
         }
     }
 

server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.ProjectScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -31,6 +32,8 @@
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Collections;
+
 public class ClusterService extends AbstractLifecycleComponent {
     private final MasterService masterService;
 
@@ -52,12 +55,31 @@ public class ClusterService extends AbstractLifecycleComponent {
 
     private final ClusterSettings clusterSettings;
 
+    private final ProjectScopedSettings projectScopedSettings;
+
     private final String nodeName;
 
     public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool, TaskManager taskManager) {
         this(
             settings,
             clusterSettings,
+            new ProjectScopedSettings(settings, Collections.emptySet()),
+            new MasterService(settings, clusterSettings, threadPool, taskManager),
+            new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool)
+        );
+    }
+
+    public ClusterService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ProjectScopedSettings projectScopedSettings,
+        ThreadPool threadPool,
+        TaskManager taskManager
+    ) {
+        this(
+            settings,
+            clusterSettings,
+            projectScopedSettings,
             new MasterService(settings, clusterSettings, threadPool, taskManager),
             new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool)
         );
@@ -68,10 +90,21 @@ public ClusterService(
         ClusterSettings clusterSettings,
         MasterService masterService,
         ClusterApplierService clusterApplierService
+    ) {
+        this(settings, clusterSettings, new ProjectScopedSettings(settings, Collections.emptySet()), masterService, clusterApplierService);
+    }
+
+    public ClusterService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ProjectScopedSettings projectScopedSettings,
+        MasterService masterService,
+        ClusterApplierService clusterApplierService
     ) {
         this.settings = settings;
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.masterService = masterService;
+        this.projectScopedSettings = projectScopedSettings;
         this.operationRouting = new OperationRouting(settings, clusterSettings);
         this.clusterSettings = clusterSettings;
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
@@ -201,6 +234,10 @@ public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
 
+    public ProjectScopedSettings getProjectScopedSettings() {
+        return projectScopedSettings;
+    }
+
     /**
      * The node's settings.
      */