Skip to content

Validate Certificate Identity provided in Cross Cluster API Key Certificate #136299

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
+559 −107
Open

Validate Certificate Identity provided in Cross Cluster API Key Certificate #136299

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
99fac10
Validate certificate identity from cross cluster creds
jfreden Oct 7, 2025
0cc2ea3
Update docs/changelog/136299.yaml
jfreden Oct 15, 2025
69c9a17
[CI] Auto commit changes from spotless
Oct 15, 2025
eb7a3f7
fixup! Pattern match
jfreden Oct 15, 2025
d63b4ec
Add pattern cache
jfreden Oct 15, 2025
e598e69
Add tests for pattern cache
jfreden Oct 16, 2025
9022c68
Merge remote-tracking branch 'upstream/main' into validate_signature_…
jfreden Oct 16, 2025
b4dfd67
fixup! logging
jfreden Oct 16, 2025
59dbd74
Update x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/s…
jfreden Oct 17, 2025
649a7e4
Update x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/s…
jfreden Oct 17, 2025
4d07535
fixup! Code review comments
jfreden Oct 17, 2025
d1b780d
Merge branch 'main' into validate_signature_identity
jfreden Oct 17, 2025
2203889
[CI] Auto commit changes from spotless
Oct 17, 2025
ed4093a
fixup! CI
jfreden Oct 17, 2025
094c15f
fixup! Import
jfreden Oct 17, 2025
a71ca11
[CI] Auto commit changes from spotless
Oct 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/136299.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 136299
summary: Validate certificate identity from cross cluster creds
area: Security
type: enhancement
issues: []
144 changes: 121 additions & 23 deletions ...g/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.junit.rules.TestRule;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
Expand All @@ -38,7 +39,7 @@

public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRemoteClusterSecurityTestCase {

private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();
private static final AtomicReference<Map<String, Object>> MY_REMOTE_API_KEY_MAP_REF = new AtomicReference<>();

static {
fulfillingCluster = ElasticsearchCluster.local()
Expand All @@ -49,8 +50,12 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
.setting("xpack.security.remote_cluster_server.ssl.enabled", "true")
.setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
.setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
.setting("xpack.security.audit.enabled", "true")
.setting(
"xpack.security.audit.logfile.events.include",
"[authentication_success, authentication_failed, access_denied, access_granted]"
)
.configFile("signing_ca.crt", Resource.fromClasspath("signing/root.crt"))
.setting("cluster.remote.signing.certificate_authorities", "signing_ca.crt")
.keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
.build();

Expand All @@ -60,22 +65,25 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
.setting("xpack.security.remote_cluster_client.ssl.enabled", "true")
.setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
.configFile("signing.crt", Resource.fromClasspath("signing/signing.crt"))
.setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
.configFile("signing.key", Resource.fromClasspath("signing/signing.key"))
.setting("cluster.remote.my_remote_cluster.signing.key", "signing.key")
.keystore("cluster.remote.my_remote_cluster.credentials", () -> {
if (API_KEY_MAP_REF.get() == null) {
final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
if (MY_REMOTE_API_KEY_MAP_REF.get() == null) {
final var accessJson = """
{
"search": [
{
"names": ["index*", "not_found_index"]
}
]
}""");
API_KEY_MAP_REF.set(apiKeyMap);
}""";
MY_REMOTE_API_KEY_MAP_REF.set(
createCrossClusterAccessApiKey(
accessJson,
randomFrom("CN=instance", "^CN=instance$", "(?i)^CN=instance$", "^CN=[A-Za-z0-9_]+$")
)
);
}
return (String) API_KEY_MAP_REF.get().get("encoded");
return (String) MY_REMOTE_API_KEY_MAP_REF.get().get("encoded");
})
.keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey())
.build();
Expand All @@ -86,33 +94,102 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster);

public void testCrossClusterSearchWithCrossClusterApiKeySigning() throws Exception {
indexTestData();
assertCrossClusterSearchSuccessfulWithResult();
updateClusterSettings(
Settings.builder()
.put("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
.put("cluster.remote.my_remote_cluster.signing.key", "signing.key")
.build()
);

// Change the CA to something that doesn't trust the signing cert
updateClusterSettingsFulfillingCluster(
Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
);
assertCrossClusterAuthFail();

// Update settings on query cluster to ignore unavailable remotes
updateClusterSettings(Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build());
indexTestData();

// Make sure we can search if cert trusted
{
assertCrossClusterSearchSuccessfulWithResult();
}

// Test CA that does not trust cert
{
// Change the CA to something that doesn't trust the signing cert
updateClusterSettingsFulfillingCluster(
Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
);
assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");

// Change the CA to the default trust store
updateClusterSettingsFulfillingCluster(Settings.builder().putNull("cluster.remote.signing.certificate_authorities").build());
assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");

// Update settings on query cluster to ignore unavailable remotes
updateClusterSettings(
Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build()
);
assertCrossClusterSearchSuccessfulWithoutResult();

// Reset skip_unavailable
updateClusterSettings(
Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(false)).build()
);

assertCrossClusterSearchSuccessfulWithoutResult();
// Reset ca cert
updateClusterSettingsFulfillingCluster(
Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
);
// Confirm reset was successful
assertCrossClusterSearchSuccessfulWithResult();
}

// Test no signature provided
{
updateClusterSettings(
Settings.builder()
.putNull("cluster.remote.my_remote_cluster.signing.certificate")
.putNull("cluster.remote.my_remote_cluster.signing.key")
.build()
);
assertCrossClusterAuthFail("Expected signature for cross cluster API key, but no signature was provided");

// TODO add test for certificate identity configured for API key but no signature provided (should 401)
// Reset
updateClusterSettings(
Settings.builder()
.put("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
.put("cluster.remote.my_remote_cluster.signing.key", "signing.key")
.build()
);
}

// TODO add test for certificate identity not configured for API key but signature provided (should 200)
// Test API key without certificate identity and send signature anyway
{
final var accessJson = """
{
"search": [
{
"names": ["index*", "not_found_index"]
}
]
}""";
MY_REMOTE_API_KEY_MAP_REF.set(createCrossClusterAccessApiKey(accessJson));
assertCrossClusterSearchSuccessfulWithResult();

// TODO add test for certificate identity not configured for API key but wrong signature provided (should 401)
// Change the CA to the default trust store to make sure untrusted signature fails auth even if it's not required
updateClusterSettingsFulfillingCluster(Settings.builder().putNull("cluster.remote.signing.certificate_authorities").build());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am curious, why fail if a certificate is invalid but not required? Is it to protect against misconfiguration?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am curious, why fail if a certificate is invalid but not required? Is it to catch misconfigurations early?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, if a signature is provided I think we should try to validate it and fail if it's not correct since it's unexpected. Better to be less lenient in this case I think.

Copy link
Contributor

@tvernum tvernum Oct 17, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It also gives customers a way to progressively switch over to signing. If they configure signing on the origin cluster, and it doesn't break, then they can rely on the fact that it's probably working correctly. Then they can turn on enforcement on the linked cluster.

And if that first step does break, then they can just turn off signing on the origin, without needing to reconfigure both sides.

assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");

// TODO add test for certificate identity regex matching (should 200)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we have this test?
I can't see it.

Copy link
Contributor Author

@jfreden jfreden Oct 17, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we do randomFrom("CN=instance", "^CN=instance$", "(?i)^CN=instance$", "^CN=[A-Za-z0-9_]+$") when we set up the API key as part of setting up the query cluster.

// Reset
updateClusterSettingsFulfillingCluster(
Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
);
}
}

private void assertCrossClusterAuthFail() {
private void assertCrossClusterAuthFail(String expectedMessage) {
var responseException = assertThrows(ResponseException.class, () -> simpleCrossClusterSearch(randomBoolean()));
assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(401));
assertThat(responseException.getMessage(), containsString("Failed to verify cross cluster api key signature certificate from [("));
assertThat(responseException.getMessage(), containsString(expectedMessage));
}

private void assertCrossClusterSearchSuccessfulWithoutResult() throws IOException {
Expand Down Expand Up @@ -227,4 +304,25 @@ private Response performRequestWithRemoteAccessUser(final Request request) throw
request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS)));
return client().performRequest(request);
}

protected static Map<String, Object> createCrossClusterAccessApiKey(String accessJson, String certificateIdentity) {
initFulfillingClusterClient();
final var createCrossClusterApiKeyRequest = new Request("POST", "/_security/cross_cluster/api_key");
createCrossClusterApiKeyRequest.setJsonEntity(Strings.format("""
{
"name": "cross_cluster_access_key",
"certificate_identity": "%s",
"access": %s
}""", certificateIdentity, accessJson));
try {
final Response createCrossClusterApiKeyResponse = performRequestWithAdminUser(
fulfillingClusterClient,
createCrossClusterApiKeyRequest
);
assertOK(createCrossClusterApiKeyResponse);
return responseAsMap(createCrossClusterApiKeyResponse);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
1 change: 1 addition & 0 deletions x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1643,6 +1643,7 @@ public static List<Setting<?>> getSettings(
settingsList.add(ApiKeyService.CACHE_MAX_KEYS_SETTING);
settingsList.add(ApiKeyService.CACHE_TTL_SETTING);
settingsList.add(ApiKeyService.DOC_CACHE_TTL_SETTING);
settingsList.add(ApiKeyService.CERTIFICATE_IDENTITY_PATTERN_CACHE_TTL_SETTING);
settingsList.add(NativePrivilegeStore.CACHE_MAX_APPLICATIONS_SETTING);
settingsList.add(NativePrivilegeStore.CACHE_TTL_SETTING);
settingsList.add(OPERATOR_PRIVILEGES_ENABLED);
Expand Down
Loading