Make project and cluster secrets customs available in server

docs/changelog/125406.yaml

@@ -0,0 +1,5 @@
+pr: 125406
+summary: Make project and cluster secrets customs available in core
+area: Security
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/module-info.java

@@ -230,6 +230,7 @@
     exports org.elasticsearch.xpack.core.watcher.trigger;
     exports org.elasticsearch.xpack.core.watcher.watch;
     exports org.elasticsearch.xpack.core.watcher;
+    exports org.elasticsearch.xpack.core.security.secrets;
 
     provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber
         with

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -92,6 +92,8 @@
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
+import org.elasticsearch.xpack.core.security.secrets.ClusterSecrets;
+import org.elasticsearch.xpack.core.security.secrets.ProjectSecrets;
 import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams;
 import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
@@ -182,6 +184,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 RemoteClusterPermissionGroup.NAME,
                 RemoteClusterPermissionGroup::new
             ),
+            // security : secrets
+            new NamedWriteableRegistry.Entry(NamedDiff.class, ClusterSecrets.TYPE, ClusterSecrets::readDiffFrom),
+            new NamedWriteableRegistry.Entry(ClusterState.Custom.class, ClusterSecrets.TYPE, ClusterSecrets::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, ProjectSecrets.TYPE, ProjectSecrets::readDiffFrom),
+            new NamedWriteableRegistry.Entry(Metadata.ProjectCustom.class, ProjectSecrets.TYPE, ProjectSecrets::new),
             // eql
             new NamedWriteableRegistry.Entry(XPackFeatureUsage.class, XPackField.EQL, EqlFeatureSetUsage::new),
             // esql

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.secrets;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.xcontent.ToXContent;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Objects;
+
+/**
+ * Secrets that are stored in cluster state
+ *
+ * <p>Cluster state secrets are initially loaded on each node, from a file on disk,
+ * in the format defined by {@link org.elasticsearch.common.settings.LocallyMountedSecrets}.
+ * Once the cluster is running, the master node watches the file for changes. This class
+ * propagates changes in the file-based secure settings from the master node out to other
+ * nodes.
+ *
+ * <p>Since the master node should always have settings on disk, we don't need to
+ * persist this class to saved cluster state, either on disk or in the cloud. Therefore,
+ * we have defined this {@link ClusterState.Custom} as a private custom object. Additionally,
+ * we don't want to ever write this class's secrets out in a client response, so
+ * {@link #toXContentChunked(ToXContent.Params)} returns an empty iterator.
+ */
+public class ClusterSecrets extends AbstractNamedDiffable<ClusterState.Custom> implements ClusterState.Custom {
+
+    /**
+     * The name for this data class
+     *
+     * <p>This name will be used to identify this {@link org.elasticsearch.common.io.stream.NamedWriteable} in cluster
+     * state. See {@link #getWriteableName()}.
+     */
+    public static final String TYPE = "cluster_state_secrets";
+
+    private final SecureClusterStateSettings settings;
+    private final long version;
+
+    public ClusterSecrets(long version, SecureClusterStateSettings settings) {
+        this.version = version;
+        this.settings = settings;
+    }
+
+    public ClusterSecrets(StreamInput in) throws IOException {
+        this.version = in.readLong();
+        this.settings = new SecureClusterStateSettings(in);
+    }
+
+    public SecureSettings getSettings() {
+        return new SecureClusterStateSettings(settings);
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    @Override
+    public boolean isPrivate() {
+        return true;
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        // never render this to the user
+        return Collections.emptyIterator();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.V_8_9_X;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(version);
+        settings.writeTo(out);
+    }
+
+    public static NamedDiff<ClusterState.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(ClusterState.Custom.class, TYPE, in);
+    }
+
+    @Override
+    public String toString() {
+        return "ClusterStateSecrets{[all secret]}";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ClusterSecrets that = (ClusterSecrets) o;
+        return version == that.version && Objects.equals(settings, that.settings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(settings, version);
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.secrets;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureSettings;
+import org.elasticsearch.xcontent.ToXContent;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Objects;
+
+/**
+ * Secrets that are stored in project state as a {@link Metadata.ProjectCustom}
+ *
+ * <p>Project state secrets are initially loaded on the master node, from a file on disk.
+ * Once the cluster is running, the master node watches the file for changes. This class
+ * propagates changes in the file-based secure settings for each project from the master
+ * node out to other nodes using the transport protocol.
+ *
+ * <p>Since the master node should always have settings on disk, we don't need to
+ * persist this class to saved cluster state, either on disk or in the cloud. Therefore,
+ * we have defined this {@link Metadata.ProjectCustom} as a "private custom" object by not
+ * serializing its content in {@link #toXContentChunked(ToXContent.Params)}.
+ */
+public class ProjectSecrets extends AbstractNamedDiffable<Metadata.ProjectCustom> implements Metadata.ProjectCustom {
+
+    public static final String TYPE = "project_state_secrets";
+
+    private final SecureClusterStateSettings settings;
+
+    public ProjectSecrets(SecureClusterStateSettings settings) {
+        this.settings = settings;
+    }
+
+    public ProjectSecrets(StreamInput in) throws IOException {
+        this.settings = new SecureClusterStateSettings(in);
+    }
+
+    public SecureSettings getSettings() {
+        return new SecureClusterStateSettings(settings);
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        // No need to persist in index or return to user, so do not serialize the secrets
+        return Collections.emptyIterator();
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.MULTI_PROJECT;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        settings.writeTo(out);
+    }
+
+    public static NamedDiff<Metadata.ProjectCustom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Metadata.ProjectCustom.class, TYPE, in);
+    }
+
+    @Override
+    public String toString() {
+        return "ProjectSecrets{[all secret]}";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ProjectSecrets that = (ProjectSecrets) o;
+        return Objects.equals(settings, that.settings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(settings);
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        return EnumSet.noneOf(Metadata.XContentContext.class);
+    }
+}