From 3c1624d11915a980cf54f0c6cc5bb8d60d9333b8 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Fri, 21 Mar 2025 14:24:52 +0100 Subject: [PATCH 1/8] Make project and cluster secrets customs available in core --- .../core/src/main/java/module-info.java | 1 + .../xpack/core/XPackClientPlugin.java | 7 + .../core/security/secrets/ClusterSecrets.java | 136 ++++++++++++++ .../core/security/secrets/ProjectSecrets.java | 157 ++++++++++++++++ .../secrets/SecureClusterStateSettings.java | 167 +++++++++++++++++ .../secrets/ClusterStateSecretsTests.java | 171 ++++++++++++++++++ .../security/secrets/ProjectSecretsTests.java | 107 +++++++++++ .../SecureClusterStateSettingsTests.java | 54 ++++++ 8 files changed, 800 insertions(+) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index c6f8376d63fa9..abc649323ce97 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -230,6 +230,7 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; + exports org.elasticsearch.xpack.core.security.secrets; provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber with diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index da235b7936270..a976264b75f87 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -92,6 +92,8 @@ import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; +import org.elasticsearch.xpack.core.security.secrets.ClusterSecrets; +import org.elasticsearch.xpack.core.security.secrets.ProjectSecrets; import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams; import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; @@ -182,6 +184,11 @@ public List getNamedWriteables() { RemoteClusterPermissionGroup.NAME, RemoteClusterPermissionGroup::new ), + // security : secrets + new NamedWriteableRegistry.Entry(NamedDiff.class, ClusterSecrets.TYPE, ClusterSecrets::readDiffFrom), + new NamedWriteableRegistry.Entry(ClusterState.Custom.class, ClusterSecrets.TYPE, ClusterSecrets::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, ProjectSecrets.TYPE, ProjectSecrets::readDiffFrom), + new NamedWriteableRegistry.Entry(Metadata.ProjectCustom.class, ProjectSecrets.TYPE, ProjectSecrets::new), // eql new NamedWriteableRegistry.Entry(XPackFeatureUsage.class, XPackField.EQL, EqlFeatureSetUsage::new), // esql diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java new file mode 100644 index 0000000000000..5f618f2812ae8 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * ELASTICSEARCH CONFIDENTIAL + * __________________ + * + * Copyright Elasticsearch B.V. All rights reserved. + * + * NOTICE: All information contained herein is, and remains + * the property of Elasticsearch B.V. and its suppliers, if any. + * The intellectual and technical concepts contained herein + * are proprietary to Elasticsearch B.V. and its suppliers and + * may be covered by U.S. and Foreign Patents, patents in + * process, and are protected by trade secret or copyright + * law. Dissemination of this information or reproduction of + * this material is strictly forbidden unless prior written + * permission is obtained from Elasticsearch B.V. + */ + +package org.elasticsearch.xpack.core.security.secrets; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.xcontent.ToXContent; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.Objects; + +/** + * Secrets that are stored in cluster state + * + *

Cluster state secrets are initially loaded on each node, from a file on disk, + * in the format defined by {@link org.elasticsearch.common.settings.LocallyMountedSecrets}. + * Once the cluster is running, the master node watches the file for changes. This class + * propagates changes in the file-based secure settings from the master node out to other + * nodes. + * + *

Since the master node should always have settings on disk, we don't need to + * persist this class to saved cluster state, either on disk or in the cloud. Therefore, + * we have defined this {@link ClusterState.Custom} as a private custom object. Additionally, + * we don't want to ever write this class's secrets out in a client response, so + * {@link #toXContentChunked(ToXContent.Params)} returns an empty iterator. + */ +public class ClusterSecrets extends AbstractNamedDiffable implements ClusterState.Custom { + + /** + * The name for this data class + * + *

This name will be used to identify this {@link org.elasticsearch.common.io.stream.NamedWriteable} in cluster + * state. See {@link #getWriteableName()}. + */ + public static final String TYPE = "cluster_state_secrets"; + + private final SecureClusterStateSettings settings; + private final long version; + + public ClusterSecrets(long version, SecureSettings settings) { + this.version = version; + this.settings = new SecureClusterStateSettings(settings); + } + + public ClusterSecrets(StreamInput in) throws IOException { + this.version = in.readLong(); + this.settings = new SecureClusterStateSettings(in); + } + + public SecureSettings getSettings() { + return new SecureClusterStateSettings(settings); + } + + public long getVersion() { + return version; + } + + @Override + public boolean isPrivate() { + return true; + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + // never render this to the user + return Collections.emptyIterator(); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.V_8_9_X; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(version); + settings.writeTo(out); + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(ClusterState.Custom.class, TYPE, in); + } + + @Override + public String toString() { + return "ClusterStateSecrets{[all secret]}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterSecrets that = (ClusterSecrets) o; + return version == that.version && Objects.equals(settings, that.settings); + } + + @Override + public int hashCode() { + return Objects.hash(settings, version); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java new file mode 100644 index 0000000000000..ed86bbd12e239 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.secrets; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Secrets that are stored in project state as a {@link Metadata.ProjectCustom} + * + *

Project state secrets are initially loaded on the master node, from a file on disk. + * Once the cluster is running, the master node watches the file for changes. This class + * propagates changes in the file-based secure settings for each project from the master + * node out to other nodes using the transport protocol. + * + *

Since the master node should always have settings on disk, we don't need to + * persist this class to saved cluster state, either on disk or in the cloud. Therefore, + * we have defined this {@link Metadata.ProjectCustom} as a "private custom" object by not + * serializing its content in {@link #toXContentChunked(ToXContent.Params)}. + */ +public class ProjectSecrets extends AbstractNamedDiffable implements Metadata.ProjectCustom { + + public static final String TYPE = "project_state_secrets"; + + private final SecureSettings settings; + + public static final ParseField STRING_SECRETS_FIELD = new ParseField("string_secrets"); + public static final ParseField FILE_SECRETS_FIELD = new ParseField("file_secrets"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser, Void> PARSER = new ConstructingObjectParser<>( + "project_secrets_parser", + a -> { + final var decoder = Base64.getDecoder(); + + Map stringSecretsMap = a[0] == null + ? Map.of() + : ((Map) a[0]).entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getBytes(StandardCharsets.UTF_8))); + + Map fileSecretsByteMap = a[1] == null + ? Map.of() + : ((Map) a[1]).entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> decoder.decode(e.getValue()))); + + Set duplicateKeys = fileSecretsByteMap.keySet() + .stream() + .filter(stringSecretsMap::containsKey) + .collect(Collectors.toSet()); + + if (duplicateKeys.isEmpty() == false) { + throw new IllegalStateException("Some settings were defined as both string and file settings: " + duplicateKeys); + } + + return Stream.concat(stringSecretsMap.entrySet().stream(), fileSecretsByteMap.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + ); + + static { + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), STRING_SECRETS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), FILE_SECRETS_FIELD); + } + + public ProjectSecrets(SecureSettings settings) { + this.settings = settings; + } + + public ProjectSecrets(StreamInput in) throws IOException { + this.settings = new SecureClusterStateSettings(in); + } + + public SecureSettings getSettings() { + return settings; + } + + public static ProjectSecrets fromXContent(XContentParser parser) { + return new ProjectSecrets(new SecureClusterStateSettings(PARSER.apply(parser, null))); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + // No need to persist in index or return to user, so do not serialize the secrets + return Collections.emptyIterator(); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.MULTI_PROJECT; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + settings.writeTo(out); + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(Metadata.ProjectCustom.class, TYPE, in); + } + + @Override + public String toString() { + return "ProjectSecrets{[all secret]}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ProjectSecrets that = (ProjectSecrets) o; + return Objects.equals(settings, that.settings); + } + + @Override + public int hashCode() { + return Objects.hash(settings); + } + + @Override + public EnumSet context() { + return EnumSet.noneOf(Metadata.XContentContext.class); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java new file mode 100644 index 0000000000000..3235189cabfdb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.secrets; + +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.common.settings.SecureString; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Implementation of {@link SecureSettings} that represents secrets in cluster state. Secrets are stored as byte arrays along with + * their SHA-256 digests. Provides functionality to read and serialize secure settings to broadcast them as part of cluster state. + * Does not provide any encryption. + */ +public class SecureClusterStateSettings implements SecureSettings { + + private final Map secrets; + + SecureClusterStateSettings(SecureSettings secureSettings) { + secrets = new HashMap<>(); + for (String key : secureSettings.getSettingNames()) { + try { + secrets.put( + key, + new Entry(getValueAsByteArray(secureSettings, key), SecureClusterStateSettings.getSHA256Digest(secureSettings, key)) + ); + } catch (IOException | GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + } + + SecureClusterStateSettings(StreamInput in) throws IOException { + secrets = in.readMap(StreamInput::readString, v -> new Entry(in.readByteArray(), in.readByteArray())); + } + + SecureClusterStateSettings(Map settings) { + secrets = settings.entrySet() + .stream() + .collect( + Collectors.toMap(Map.Entry::getKey, entry -> new Entry(entry.getValue(), MessageDigests.sha256().digest(entry.getValue()))) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(secrets, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + } + + @Override + public boolean isLoaded() { + return true; + } + + @Override + public Set getSettingNames() { + return secrets.keySet(); + } + + @Override + public SecureString getString(String setting) { + Entry value = secrets.get(setting); + if (value == null) { + return null; + } + ByteBuffer byteBuffer = ByteBuffer.wrap(value.secret()); + CharBuffer charBuffer = StandardCharsets.UTF_8.decode(byteBuffer); + return new SecureString(Arrays.copyOfRange(charBuffer.array(), charBuffer.position(), charBuffer.limit())); + } + + @Override + public InputStream getFile(String setting) { + var value = secrets.get(setting); + if (value == null) { + return null; + } + return new ByteArrayInputStream(value.secret()); + } + + @Override + public byte[] getSHA256Digest(String setting) { + return secrets.get(setting).sha256Digest(); + } + + @Override + public void close() { + if (null != secrets && secrets.isEmpty() == false) { + for (var entry : secrets.entrySet()) { + entry.setValue(null); + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SecureClusterStateSettings secrets1 = (SecureClusterStateSettings) o; + return Objects.equals(secrets, secrets1.secrets); + } + + @Override + public int hashCode() { + return Objects.hash(secrets); + } + + private static byte[] getValueAsByteArray(SecureSettings secureSettings, String key) throws GeneralSecurityException, IOException { + return secureSettings.getFile(key).readAllBytes(); + } + + private static byte[] getSHA256Digest(SecureSettings secureSettings, String key) { + try { + return secureSettings.getSHA256Digest(key); + } catch (GeneralSecurityException e) { + throw new RuntimeException(e); + } + } + + record Entry(byte[] secret, byte[] sha256Digest) implements Writeable { + + Entry(StreamInput in) throws IOException { + this(in.readByteArray(), in.readByteArray()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Entry entry = (Entry) o; + return Arrays.equals(secret, entry.secret) && Arrays.equals(sha256Digest, entry.sha256Digest); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(secret); + result = 31 * result + Arrays.hashCode(sha256Digest); + return result; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByteArray(secret); + out.writeByteArray(sha256Digest); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java new file mode 100644 index 0000000000000..80a5a74cfa9b9 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.secrets; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.LocallyMountedSecrets; +import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.AbstractNamedWriteableTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.nullValue; + +public class ClusterStateSecretsTests extends AbstractNamedWriteableTestCase { + + private static final String MOUNTED_SETTINGS = """ + { + "metadata": { + "version": "1", + "compatibility": "8.7.0" + }, + "secrets": { + "foo": "bar", + "goo": "baz" + } + } + """; + + private LocallyMountedSecrets locallyMountedSecrets; + + @Before + public void setUp() throws Exception { + super.setUp(); + + Environment environment = newEnvironment(); + writeTestFile(environment.configDir().resolve("secrets").resolve("secrets.json"), MOUNTED_SETTINGS); + locallyMountedSecrets = new LocallyMountedSecrets(environment); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + locallyMountedSecrets.close(); + } + + public void testGetSettings() throws Exception { + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + + assertThat(clusterStateSecrets.getSettings().getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(clusterStateSecrets.getSettings().getString("foo").toString(), equalTo("bar")); + assertThat(clusterStateSecrets.getSettings().getString("goo").toString(), equalTo("baz")); + + locallyMountedSecrets.close(); + + assertThat(clusterStateSecrets.getSettings().getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(clusterStateSecrets.getSettings().getString("foo").toString(), equalTo("bar")); + assertThat(clusterStateSecrets.getSettings().getString("goo").toString(), equalTo("baz")); + } + + public void testFileSettings() throws Exception { + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + + assertThat(clusterStateSecrets.getSettings().getFile("foo").readAllBytes(), equalTo("bar".getBytes(StandardCharsets.UTF_8))); + } + + public void testSerialize() throws Exception { + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + + final BytesStreamOutput out = new BytesStreamOutput(); + clusterStateSecrets.writeTo(out); + final ClusterSecrets fromStream = new ClusterSecrets(out.bytes().streamInput()); + + assertThat(fromStream.getVersion(), equalTo(clusterStateSecrets.getVersion())); + + assertThat(fromStream.getSettings().getSettingNames(), hasSize(2)); + assertThat(fromStream.getSettings().getSettingNames(), containsInAnyOrder("foo", "goo")); + + assertEquals(clusterStateSecrets.getSettings().getString("foo"), fromStream.getSettings().getString("foo")); + assertTrue(fromStream.getSettings().isLoaded()); + } + + public void testToXContentChunked() { + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + + // we never serialize anything to x-content + assertFalse(clusterStateSecrets.toXContentChunked(EMPTY_PARAMS).hasNext()); + } + + public void testToString() { + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + + assertThat(clusterStateSecrets.toString(), equalTo("ClusterStateSecrets{[all secret]}")); + } + + public void testClose() throws Exception { + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + + SecureSettings settings = clusterStateSecrets.getSettings(); + assertThat(settings.getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(settings.getString("foo").toString(), equalTo("bar")); + assertThat(settings.getString("goo").toString(), equalTo("baz")); + + settings.close(); + + // we can close the copy + assertThat(settings.getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(settings.getString("foo"), nullValue()); + assertThat(settings.getString("goo"), nullValue()); + + // fetching again returns a fresh object + SecureSettings settings2 = clusterStateSecrets.getSettings(); + assertThat(settings2.getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(settings2.getString("foo").toString(), equalTo("bar")); + assertThat(settings2.getString("goo").toString(), equalTo("baz")); + } + + @Override + protected ClusterSecrets createTestInstance() { + return new ClusterSecrets( + randomLong(), + new SecureClusterStateSettings( + randomMap(0, 3, () -> Tuple.tuple(randomAlphaOfLength(10), randomAlphaOfLength(15).getBytes(Charset.defaultCharset()))) + ) + ); + } + + @Override + protected ClusterSecrets mutateInstance(ClusterSecrets instance) throws IOException { + return new ClusterSecrets(instance.getVersion() + 1L, instance.getSettings()); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry( + List.of(new NamedWriteableRegistry.Entry(ClusterSecrets.class, ClusterSecrets.TYPE, ClusterSecrets::new)) + ); + } + + @Override + protected Class categoryClass() { + return ClusterSecrets.class; + } + + private void writeTestFile(Path path, String contents) throws IOException { + Path tempFilePath = createTempFile(); + + Files.writeString(tempFilePath, contents); + Files.createDirectories(path.getParent()); + Files.move(tempFilePath, path, StandardCopyOption.ATOMIC_MOVE); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java new file mode 100644 index 0000000000000..60b9c47c868a6 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.secrets; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.test.AbstractNamedWriteableTestCase; +import org.junit.Before; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class ProjectSecretsTests extends AbstractNamedWriteableTestCase { + + private SecureClusterStateSettings secureClusterStateSettings; + + @Before + public void setUp() throws Exception { + super.setUp(); + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + // SecureSettings in cluster state are handled as file settings (get the byte array) both can be fetched as + // string or file + mockSecureSettings.setFile("foo", "bar".getBytes(StandardCharsets.UTF_8)); + mockSecureSettings.setFile("goo", "baz".getBytes(StandardCharsets.UTF_8)); + secureClusterStateSettings = new SecureClusterStateSettings(mockSecureSettings); + } + + public void testGetSettings() throws Exception { + ProjectSecrets projectSecrets = new ProjectSecrets(secureClusterStateSettings); + assertThat(projectSecrets.getSettings().getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(projectSecrets.getSettings().getString("foo").toString(), equalTo("bar")); + assertThat(new String(projectSecrets.getSettings().getFile("goo").readAllBytes(), StandardCharsets.UTF_8), equalTo("baz")); + } + + public void testSerialize() throws Exception { + ProjectSecrets projectSecrets = new ProjectSecrets(secureClusterStateSettings); + + final BytesStreamOutput out = new BytesStreamOutput(); + projectSecrets.writeTo(out); + final ProjectSecrets fromStream = new ProjectSecrets(out.bytes().streamInput()); + + assertThat(fromStream.getSettings().getSettingNames(), hasSize(2)); + assertThat(fromStream.getSettings().getSettingNames(), containsInAnyOrder("foo", "goo")); + + assertEquals(projectSecrets.getSettings().getString("foo"), fromStream.getSettings().getString("foo")); + assertThat(new String(projectSecrets.getSettings().getFile("goo").readAllBytes(), StandardCharsets.UTF_8), equalTo("baz")); + assertTrue(fromStream.getSettings().isLoaded()); + } + + public void testToXContentChunked() { + ProjectSecrets projectSecrets = new ProjectSecrets(secureClusterStateSettings); + // we never serialize anything to x-content + assertFalse(projectSecrets.toXContentChunked(EMPTY_PARAMS).hasNext()); + } + + public void testToString() { + ProjectSecrets projectSecrets = new ProjectSecrets(secureClusterStateSettings); + assertThat(projectSecrets.toString(), equalTo("ProjectSecrets{[all secret]}")); + } + + @Override + protected ProjectSecrets createTestInstance() { + return new ProjectSecrets( + new SecureClusterStateSettings( + randomMap(0, 3, () -> Tuple.tuple(randomAlphaOfLength(10), randomAlphaOfLength(15).getBytes(Charset.defaultCharset()))) + ) + ); + } + + @Override + protected ProjectSecrets mutateInstance(ProjectSecrets instance) { + Map updatedSettings = new HashMap<>(); + + for (var settingName : instance.getSettings().getSettingNames()) { + updatedSettings.put(settingName, instance.getSettings().getString(settingName).toString().getBytes(StandardCharsets.UTF_8)); + } + updatedSettings.put(randomAlphaOfLength(9), randomAlphaOfLength(14).getBytes(StandardCharsets.UTF_8)); + return new ProjectSecrets(new SecureClusterStateSettings(updatedSettings)); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry( + List.of(new NamedWriteableRegistry.Entry(ProjectSecrets.class, ProjectSecrets.TYPE, ProjectSecrets::new)) + ); + } + + @Override + protected Class categoryClass() { + return ProjectSecrets.class; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java new file mode 100644 index 0000000000000..994f32a34f44f --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.secrets; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class SecureClusterStateSettingsTests extends ESTestCase { + MockSecureSettings mockSecureSettings = new MockSecureSettings(); + + @Before + public void setUp() throws Exception { + super.setUp(); + // SecureSettings in cluster state are handled as file settings (get the byte array) both can be fetched as + // string or file + mockSecureSettings.setFile("foo", "bar".getBytes(StandardCharsets.UTF_8)); + mockSecureSettings.setFile("goo", "baz".getBytes(StandardCharsets.UTF_8)); + } + + public void testGetSettings() throws Exception { + SecureClusterStateSettings secureClusterStateSettings = new SecureClusterStateSettings(mockSecureSettings); + assertThat(secureClusterStateSettings.getSettingNames(), containsInAnyOrder("foo", "goo")); + assertThat(secureClusterStateSettings.getString("foo").toString(), equalTo("bar")); + assertThat(new String(secureClusterStateSettings.getFile("goo").readAllBytes(), StandardCharsets.UTF_8), equalTo("baz")); + } + + public void testSerialize() throws Exception { + SecureClusterStateSettings secureClusterStateSettings = new SecureClusterStateSettings(mockSecureSettings); + + final BytesStreamOutput out = new BytesStreamOutput(); + secureClusterStateSettings.writeTo(out); + final SecureClusterStateSettings fromStream = new SecureClusterStateSettings(out.bytes().streamInput()); + + assertThat(fromStream.getSettingNames(), hasSize(2)); + assertThat(fromStream.getSettingNames(), containsInAnyOrder("foo", "goo")); + + assertEquals(secureClusterStateSettings.getString("foo"), fromStream.getString("foo")); + assertThat(new String(fromStream.getFile("goo").readAllBytes(), StandardCharsets.UTF_8), equalTo("baz")); + assertTrue(fromStream.isLoaded()); + } +} From 793073ff38bcf8cef18c3c34b58f2fd13e361c96 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Mon, 24 Mar 2025 09:11:07 +0100 Subject: [PATCH 2/8] fixup! License header --- .../core/security/secrets/ClusterSecrets.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java index 5f618f2812ae8..077cef95bfa46 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java @@ -5,23 +5,6 @@ * 2.0. */ -/* - * ELASTICSEARCH CONFIDENTIAL - * __________________ - * - * Copyright Elasticsearch B.V. All rights reserved. - * - * NOTICE: All information contained herein is, and remains - * the property of Elasticsearch B.V. and its suppliers, if any. - * The intellectual and technical concepts contained herein - * are proprietary to Elasticsearch B.V. and its suppliers and - * may be covered by U.S. and Foreign Patents, patents in - * process, and are protected by trade secret or copyright - * law. Dissemination of this information or reproduction of - * this material is strictly forbidden unless prior written - * permission is obtained from Elasticsearch B.V. - */ - package org.elasticsearch.xpack.core.security.secrets; import org.elasticsearch.TransportVersion; From 38c23048a90be40a887118bf4e826bacd471486c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Mon, 24 Mar 2025 10:55:18 +0100 Subject: [PATCH 3/8] Update docs/changelog/125406.yaml --- docs/changelog/125406.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/125406.yaml diff --git a/docs/changelog/125406.yaml b/docs/changelog/125406.yaml new file mode 100644 index 0000000000000..cbf02d56ac6b3 --- /dev/null +++ b/docs/changelog/125406.yaml @@ -0,0 +1,5 @@ +pr: 125406 +summary: Make project and cluster secrets customs available in core +area: Security +type: enhancement +issues: [] From 74b5535bb6df2589d33a4b515259a25fd244e148 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Tue, 25 Mar 2025 09:29:19 +0100 Subject: [PATCH 4/8] Make interfaces more consistent --- .../core/security/secrets/ClusterSecrets.java | 6 +- .../core/security/secrets/ProjectSecrets.java | 57 +------------------ .../secrets/SecureClusterStateSettings.java | 12 ++-- .../secrets/ClusterStateSecretsTests.java | 14 ++--- 4 files changed, 18 insertions(+), 71 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java index 077cef95bfa46..54d56f2b5b4ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java @@ -50,9 +50,9 @@ public class ClusterSecrets extends AbstractNamedDiffable i private final SecureClusterStateSettings settings; private final long version; - public ClusterSecrets(long version, SecureSettings settings) { + public ClusterSecrets(long version, SecureClusterStateSettings settings) { this.version = version; - this.settings = new SecureClusterStateSettings(settings); + this.settings = settings; } public ClusterSecrets(StreamInput in) throws IOException { @@ -61,7 +61,7 @@ public ClusterSecrets(StreamInput in) throws IOException { } public SecureSettings getSettings() { - return new SecureClusterStateSettings(settings); + return settings; } public long getVersion() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java index ed86bbd12e239..3fe305ce4b271 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java @@ -15,22 +15,13 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.SecureSettings; -import org.elasticsearch.xcontent.ConstructingObjectParser; -import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.Collections; import java.util.EnumSet; import java.util.Iterator; -import java.util.Map; import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Secrets that are stored in project state as a {@link Metadata.ProjectCustom} @@ -49,49 +40,9 @@ public class ProjectSecrets extends AbstractNamedDiffable, Void> PARSER = new ConstructingObjectParser<>( - "project_secrets_parser", - a -> { - final var decoder = Base64.getDecoder(); - - Map stringSecretsMap = a[0] == null - ? Map.of() - : ((Map) a[0]).entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getBytes(StandardCharsets.UTF_8))); - - Map fileSecretsByteMap = a[1] == null - ? Map.of() - : ((Map) a[1]).entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> decoder.decode(e.getValue()))); - - Set duplicateKeys = fileSecretsByteMap.keySet() - .stream() - .filter(stringSecretsMap::containsKey) - .collect(Collectors.toSet()); - - if (duplicateKeys.isEmpty() == false) { - throw new IllegalStateException("Some settings were defined as both string and file settings: " + duplicateKeys); - } - - return Stream.concat(stringSecretsMap.entrySet().stream(), fileSecretsByteMap.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - ); - - static { - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), STRING_SECRETS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), FILE_SECRETS_FIELD); - } + private final SecureClusterStateSettings settings; - public ProjectSecrets(SecureSettings settings) { + public ProjectSecrets(SecureClusterStateSettings settings) { this.settings = settings; } @@ -103,10 +54,6 @@ public SecureSettings getSettings() { return settings; } - public static ProjectSecrets fromXContent(XContentParser parser) { - return new ProjectSecrets(new SecureClusterStateSettings(PARSER.apply(parser, null))); - } - @Override public Iterator toXContentChunked(ToXContent.Params params) { // No need to persist in index or return to user, so do not serialize the secrets diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java index 3235189cabfdb..5e55f6aa6d18f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java @@ -37,7 +37,7 @@ public class SecureClusterStateSettings implements SecureSettings { private final Map secrets; - SecureClusterStateSettings(SecureSettings secureSettings) { + public SecureClusterStateSettings(SecureSettings secureSettings) { secrets = new HashMap<>(); for (String key : secureSettings.getSettingNames()) { try { @@ -51,11 +51,7 @@ public class SecureClusterStateSettings implements SecureSettings { } } - SecureClusterStateSettings(StreamInput in) throws IOException { - secrets = in.readMap(StreamInput::readString, v -> new Entry(in.readByteArray(), in.readByteArray())); - } - - SecureClusterStateSettings(Map settings) { + public SecureClusterStateSettings(Map settings) { secrets = settings.entrySet() .stream() .collect( @@ -63,6 +59,10 @@ public class SecureClusterStateSettings implements SecureSettings { ); } + SecureClusterStateSettings(StreamInput in) throws IOException { + secrets = in.readMap(StreamInput::readString, v -> new Entry(in.readByteArray(), in.readByteArray())); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(secrets, StreamOutput::writeString, (o, v) -> v.writeTo(o)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java index 80a5a74cfa9b9..67b760f9a3470 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java @@ -64,7 +64,7 @@ public void tearDown() throws Exception { } public void testGetSettings() throws Exception { - ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, new SecureClusterStateSettings(locallyMountedSecrets)); assertThat(clusterStateSecrets.getSettings().getSettingNames(), containsInAnyOrder("foo", "goo")); assertThat(clusterStateSecrets.getSettings().getString("foo").toString(), equalTo("bar")); @@ -78,13 +78,13 @@ public void testGetSettings() throws Exception { } public void testFileSettings() throws Exception { - ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, new SecureClusterStateSettings(locallyMountedSecrets)); assertThat(clusterStateSecrets.getSettings().getFile("foo").readAllBytes(), equalTo("bar".getBytes(StandardCharsets.UTF_8))); } public void testSerialize() throws Exception { - ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, new SecureClusterStateSettings(locallyMountedSecrets)); final BytesStreamOutput out = new BytesStreamOutput(); clusterStateSecrets.writeTo(out); @@ -100,20 +100,20 @@ public void testSerialize() throws Exception { } public void testToXContentChunked() { - ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, new SecureClusterStateSettings(locallyMountedSecrets)); // we never serialize anything to x-content assertFalse(clusterStateSecrets.toXContentChunked(EMPTY_PARAMS).hasNext()); } public void testToString() { - ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, new SecureClusterStateSettings(locallyMountedSecrets)); assertThat(clusterStateSecrets.toString(), equalTo("ClusterStateSecrets{[all secret]}")); } public void testClose() throws Exception { - ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, locallyMountedSecrets); + ClusterSecrets clusterStateSecrets = new ClusterSecrets(1, new SecureClusterStateSettings(locallyMountedSecrets)); SecureSettings settings = clusterStateSecrets.getSettings(); assertThat(settings.getSettingNames(), containsInAnyOrder("foo", "goo")); @@ -146,7 +146,7 @@ protected ClusterSecrets createTestInstance() { @Override protected ClusterSecrets mutateInstance(ClusterSecrets instance) throws IOException { - return new ClusterSecrets(instance.getVersion() + 1L, instance.getSettings()); + return new ClusterSecrets(instance.getVersion() + 1L, new SecureClusterStateSettings(instance.getSettings())); } @Override From a34e039e35a5f519e6aad7bada0360ca4133f288 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Tue, 25 Mar 2025 10:04:20 +0100 Subject: [PATCH 5/8] fixup! Test --- .../xpack/core/security/secrets/ClusterSecrets.java | 2 +- .../xpack/core/security/secrets/ProjectSecrets.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java index 54d56f2b5b4ca..3bffb83a886fd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java @@ -61,7 +61,7 @@ public ClusterSecrets(StreamInput in) throws IOException { } public SecureSettings getSettings() { - return settings; + return new SecureClusterStateSettings(settings); } public long getVersion() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java index 3fe305ce4b271..1fb93a9a42b37 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java @@ -51,7 +51,7 @@ public ProjectSecrets(StreamInput in) throws IOException { } public SecureSettings getSettings() { - return settings; + return new SecureClusterStateSettings(settings); } @Override From 42a6ba7a8d90496443fb64e15df2a39a87206f97 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Tue, 25 Mar 2025 14:00:54 +0100 Subject: [PATCH 6/8] Move to server --- docs/changelog/125406.yaml | 5 ----- .../org/elasticsearch/cluster/ClusterModule.java | 6 +++++- .../common/settings}/ClusterSecrets.java | 11 ++++++----- .../common/settings}/ProjectSecrets.java | 11 ++++++----- .../common/settings}/SecureClusterStateSettings.java | 12 ++++++------ .../common/settings}/ClusterStateSecretsTests.java | 12 ++++++++---- .../common/settings}/ProjectSecretsTests.java | 12 ++++++++---- .../settings}/SecureClusterStateSettingsTests.java | 11 +++++++---- x-pack/plugin/core/src/main/java/module-info.java | 1 - .../elasticsearch/xpack/core/XPackClientPlugin.java | 9 ++------- 10 files changed, 48 insertions(+), 42 deletions(-) delete mode 100644 docs/changelog/125406.yaml rename {x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets => server/src/main/java/org/elasticsearch/common/settings}/ClusterSecrets.java (89%) rename {x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets => server/src/main/java/org/elasticsearch/common/settings}/ProjectSecrets.java (88%) rename {x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets => server/src/main/java/org/elasticsearch/common/settings}/SecureClusterStateSettings.java (92%) rename {x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets => server/src/test/java/org/elasticsearch/common/settings}/ClusterStateSecretsTests.java (92%) rename {x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets => server/src/test/java/org/elasticsearch/common/settings}/ProjectSecretsTests.java (89%) rename {x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets => server/src/test/java/org/elasticsearch/common/settings}/SecureClusterStateSettingsTests.java (82%) diff --git a/docs/changelog/125406.yaml b/docs/changelog/125406.yaml deleted file mode 100644 index cbf02d56ac6b3..0000000000000 --- a/docs/changelog/125406.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 125406 -summary: Make project and cluster secrets customs available in core -area: Security -type: enhancement -issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 1e982ca25e1a4..c5fd400070936 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -65,7 +65,9 @@ import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.settings.ClusterSecrets; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.ProjectSecrets; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -264,7 +266,9 @@ public static List getNamedWriteables() { RegisteredPolicySnapshots::new, RegisteredPolicySnapshots.RegisteredSnapshotsDiff::new ); - + // Secrets + registerClusterCustom(entries, ClusterSecrets.TYPE, ClusterSecrets::new, ClusterSecrets::readDiffFrom); + registerProjectCustom(entries, ProjectSecrets.TYPE, ProjectSecrets::new, ProjectSecrets::readDiffFrom); // Task Status (not Diffable) entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSecrets.java similarity index 89% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java rename to server/src/main/java/org/elasticsearch/common/settings/ClusterSecrets.java index 3bffb83a886fd..d8039b59932aa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ClusterSecrets.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSecrets.java @@ -1,11 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.security.secrets; +package org.elasticsearch.common.settings; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -14,7 +16,6 @@ import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java b/server/src/main/java/org/elasticsearch/common/settings/ProjectSecrets.java similarity index 88% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java rename to server/src/main/java/org/elasticsearch/common/settings/ProjectSecrets.java index 1fb93a9a42b37..a821e276fb64e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecrets.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ProjectSecrets.java @@ -1,11 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.security.secrets; +package org.elasticsearch.common.settings; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; @@ -14,7 +16,6 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java b/server/src/main/java/org/elasticsearch/common/settings/SecureClusterStateSettings.java similarity index 92% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java rename to server/src/main/java/org/elasticsearch/common/settings/SecureClusterStateSettings.java index 5e55f6aa6d18f..a45d79e4acbe1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SecureClusterStateSettings.java @@ -1,18 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.security.secrets; +package org.elasticsearch.common.settings; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.SecureSettings; -import org.elasticsearch.common.settings.SecureString; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java similarity index 92% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java rename to server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java index 67b760f9a3470..eba37ff8eda28 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ClusterStateSecretsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java @@ -1,16 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.security.secrets; +package org.elasticsearch.common.settings; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.LocallyMountedSecrets; import org.elasticsearch.common.settings.SecureSettings; +import org.elasticsearch.common.settings.ClusterSecrets; +import org.elasticsearch.common.settings.SecureClusterStateSettings; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.test.AbstractNamedWriteableTestCase; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java similarity index 89% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java rename to server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java index 60b9c47c868a6..9f3b89c9989d6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/ProjectSecretsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java @@ -1,15 +1,19 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.security.secrets; +package org.elasticsearch.common.settings; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.ProjectSecrets; +import org.elasticsearch.common.settings.SecureClusterStateSettings; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.AbstractNamedWriteableTestCase; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java similarity index 82% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java rename to server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java index 994f32a34f44f..a275c7519fc3c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/secrets/SecureClusterStateSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java @@ -1,14 +1,17 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.security.secrets; +package org.elasticsearch.common.settings; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.SecureClusterStateSettings; import org.elasticsearch.test.ESTestCase; import org.junit.Before; diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index abc649323ce97..c6f8376d63fa9 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -230,7 +230,6 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; - exports org.elasticsearch.xpack.core.security.secrets; provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber with diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a976264b75f87..014f326b91bfe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -92,8 +92,8 @@ import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; -import org.elasticsearch.xpack.core.security.secrets.ClusterSecrets; -import org.elasticsearch.xpack.core.security.secrets.ProjectSecrets; +import org.elasticsearch.common.settings.ClusterSecrets; +import org.elasticsearch.common.settings.ProjectSecrets; import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams; import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; @@ -184,11 +184,6 @@ public List getNamedWriteables() { RemoteClusterPermissionGroup.NAME, RemoteClusterPermissionGroup::new ), - // security : secrets - new NamedWriteableRegistry.Entry(NamedDiff.class, ClusterSecrets.TYPE, ClusterSecrets::readDiffFrom), - new NamedWriteableRegistry.Entry(ClusterState.Custom.class, ClusterSecrets.TYPE, ClusterSecrets::new), - new NamedWriteableRegistry.Entry(NamedDiff.class, ProjectSecrets.TYPE, ProjectSecrets::readDiffFrom), - new NamedWriteableRegistry.Entry(Metadata.ProjectCustom.class, ProjectSecrets.TYPE, ProjectSecrets::new), // eql new NamedWriteableRegistry.Entry(XPackFeatureUsage.class, XPackField.EQL, EqlFeatureSetUsage::new), // esql From f16ef136e797d9ae9ff4d92f5d77cadf88bf0e57 Mon Sep 17 00:00:00 2001 From: Johannes Freden Jansson Date: Tue, 25 Mar 2025 14:02:43 +0100 Subject: [PATCH 7/8] fixup! Import --- .../java/org/elasticsearch/xpack/core/XPackClientPlugin.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 014f326b91bfe..da235b7936270 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -92,8 +92,6 @@ import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; -import org.elasticsearch.common.settings.ClusterSecrets; -import org.elasticsearch.common.settings.ProjectSecrets; import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams; import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage; import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata; From a406e6ca04ed2afe0d9a2977c13ff1778fc510eb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 25 Mar 2025 13:11:50 +0000 Subject: [PATCH 8/8] [CI] Auto commit changes from spotless --- .../common/settings/ClusterStateSecretsTests.java | 4 ---- .../elasticsearch/common/settings/ProjectSecretsTests.java | 3 --- .../common/settings/SecureClusterStateSettingsTests.java | 2 -- 3 files changed, 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java index eba37ff8eda28..edef763cc1418 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ClusterStateSecretsTests.java @@ -11,10 +11,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.LocallyMountedSecrets; -import org.elasticsearch.common.settings.SecureSettings; -import org.elasticsearch.common.settings.ClusterSecrets; -import org.elasticsearch.common.settings.SecureClusterStateSettings; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.test.AbstractNamedWriteableTestCase; diff --git a/server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java index 9f3b89c9989d6..0bfdb5a603475 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ProjectSecretsTests.java @@ -11,9 +11,6 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.ProjectSecrets; -import org.elasticsearch.common.settings.SecureClusterStateSettings; import org.elasticsearch.core.Tuple; import org.elasticsearch.test.AbstractNamedWriteableTestCase; import org.junit.Before; diff --git a/server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java index a275c7519fc3c..71414caa3ac98 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SecureClusterStateSettingsTests.java @@ -10,8 +10,6 @@ package org.elasticsearch.common.settings; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.SecureClusterStateSettings; import org.elasticsearch.test.ESTestCase; import org.junit.Before;