Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/135674.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 135674
summary: Send cross cluster api key signature as headers
area: Security
type: enhancement
issues: []
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9187000
2 changes: 1 addition & 1 deletion server/src/main/resources/transport/upper_bounds/9.3.csv
Original file line number Diff line number Diff line change
@@ -1 +1 @@
esql_plan_with_no_columns,9186000
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New transport version needed for make sure we don't serialize the new thread context header for a fulfilling cluster that doesn't support it.

add_cross_cluster_api_key_signature,9187000
Original file line number Diff line number Diff line change
Expand Up @@ -76,16 +76,6 @@ public void writeToContext(final ThreadContext ctx) throws IOException {
ctx.putHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY, encode());
}

public static CrossClusterAccessSubjectInfo readFromContext(final ThreadContext ctx) throws IOException {
final String header = ctx.getHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY);
if (header == null) {
throw new IllegalArgumentException(
"cross cluster access header [" + CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY + "] is required"
);
}
return decode(header);
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need the encoded header for the signature so I moved this to CrossClusterAccessHeaders.


public Authentication getAuthentication() {
return authentication;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ public void testWriteReadContextRoundtrip() throws IOException {
);

expectedCrossClusterAccessSubjectInfo.writeToContext(ctx);
final CrossClusterAccessSubjectInfo actual = CrossClusterAccessSubjectInfo.readFromContext(ctx);
final CrossClusterAccessSubjectInfo actual = CrossClusterAccessSubjectInfo.decode(
ctx.getHeader(CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY)
);

assertThat(actual.getAuthentication(), equalTo(expectedCrossClusterAccessSubjectInfo.getAuthentication()));
final List<Set<RoleDescriptor>> roleDescriptorsList = new ArrayList<>();
Expand All @@ -70,17 +72,6 @@ public void testRoleDescriptorsBytesToRoleDescriptors() throws IOException {
assertThat(actualRoleDescriptors, equalTo(expectedRoleDescriptors));
}

public void testThrowsOnMissingEntry() {
var actual = expectThrows(
IllegalArgumentException.class,
() -> CrossClusterAccessSubjectInfo.readFromContext(new ThreadContext(Settings.EMPTY))
);
assertThat(
actual.getMessage(),
equalTo("cross cluster access header [" + CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY + "] is required")
);
}

public void testCleanWithValidationForApiKeys() {
final Map<String, Object> initialMetadata = newHashMapWithRandomMetadata();
final AuthenticationTestHelper.AuthenticationTestBuilder builder = AuthenticationTestHelper.builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.cluster.util.resource.Resource;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
Expand Down Expand Up @@ -54,6 +55,10 @@ public class RemoteClusterSecurityBWCToRCS2ClusterRestIT extends AbstractRemoteC
.apply(commonClusterConfig)
.setting("xpack.security.remote_cluster_client.ssl.enabled", "true")
.setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
.configFile("signing.crt", Resource.fromClasspath("signing/signing.crt"))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This adds a signature config to the query cluster to make sure we don't send it when communicating with older fulfilling clusters.

.setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
.configFile("signing.key", Resource.fromClasspath("signing/signing.key"))
.setting("cluster.remote.my_remote_cluster.signing.key", "signing.key")
.keystore("cluster.remote.my_remote_cluster.credentials", () -> {
if (API_KEY_MAP_REF.get() == null) {
final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.remotecluster;

import io.netty.handler.codec.http.HttpMethod;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Strings;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchResponseUtils;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.util.resource.Resource;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;

public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRemoteClusterSecurityTestCase {

private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();

static {
fulfillingCluster = ElasticsearchCluster.local()
.name("fulfilling-cluster")
.apply(commonClusterConfig)
.setting("remote_cluster_server.enabled", "true")
.setting("remote_cluster.port", "0")
.setting("xpack.security.remote_cluster_server.ssl.enabled", "true")
.setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
.setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
.configFile("signing_ca.crt", Resource.fromClasspath("signing/root.crt"))
.setting("cluster.remote.signing.certificate_authorities", "signing_ca.crt")
.keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
.build();

queryCluster = ElasticsearchCluster.local()
.name("query-cluster")
.apply(commonClusterConfig)
.setting("xpack.security.remote_cluster_client.ssl.enabled", "true")
.setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
.configFile("signing.crt", Resource.fromClasspath("signing/signing.crt"))
.setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
.configFile("signing.key", Resource.fromClasspath("signing/signing.key"))
.setting("cluster.remote.my_remote_cluster.signing.key", "signing.key")
.keystore("cluster.remote.my_remote_cluster.credentials", () -> {
if (API_KEY_MAP_REF.get() == null) {
final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
{
"search": [
{
"names": ["index*", "not_found_index"]
}
]
}""");
API_KEY_MAP_REF.set(apiKeyMap);
}
return (String) API_KEY_MAP_REF.get().get("encoded");
})
.keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey())
.build();
}

@ClassRule
// Use a RuleChain to ensure that fulfilling cluster is started before query cluster
public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster);

public void testCrossClusterSearchWithCrossClusterApiKeySigning() throws Exception {
indexTestData();
assertCrossClusterSearchSuccessfulWithResult();

// Change the CA to something that doesn't trust the signing cert
updateClusterSettingsFulfillingCluster(
Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
);
assertCrossClusterAuthFail();

// Update settings on query cluster to ignore unavailable remotes
updateClusterSettings(Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build());

assertCrossClusterSearchSuccessfulWithoutResult();

// TODO add test for certificate identity configured for API key but no signature provided (should 401)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the cross cluster api key with a cert identity is available these test will be implemented here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll do this in a separate PR to keep scope manageable.


// TODO add test for certificate identity not configured for API key but signature provided (should 200)

// TODO add test for certificate identity not configured for API key but wrong signature provided (should 401)

// TODO add test for certificate identity regex matching (should 200)
}

private void assertCrossClusterAuthFail() {
var responseException = assertThrows(ResponseException.class, () -> simpleCrossClusterSearch(randomBoolean()));
assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(401));
assertThat(responseException.getMessage(), containsString("Failed to verify cross cluster api key signature certificate from [("));
}

private void assertCrossClusterSearchSuccessfulWithoutResult() throws IOException {
boolean alsoSearchLocally = randomBoolean();
final Response response = simpleCrossClusterSearch(alsoSearchLocally);
assertOK(response);
}

private void assertCrossClusterSearchSuccessfulWithResult() throws IOException {
boolean alsoSearchLocally = randomBoolean();
final Response response = simpleCrossClusterSearch(alsoSearchLocally);
assertOK(response);
final SearchResponse searchResponse;
try (var parser = responseAsParser(response)) {
searchResponse = SearchResponseUtils.parseSearchResponse(parser);
}
try {
final List<String> actualIndices = Arrays.stream(searchResponse.getHits().getHits())
.map(SearchHit::getIndex)
.collect(Collectors.toList());
if (alsoSearchLocally) {
assertThat(actualIndices, containsInAnyOrder("index1", "local_index"));
} else {
assertThat(actualIndices, containsInAnyOrder("index1"));
}
} finally {
searchResponse.decRef();
}
}

private Response simpleCrossClusterSearch(boolean alsoSearchLocally) throws IOException {
final var searchRequest = new Request(
"GET",
String.format(
Locale.ROOT,
"/%s%s:%s/_search?ccs_minimize_roundtrips=%s",
alsoSearchLocally ? "local_index," : "",
randomFrom("my_remote_cluster", "*", "my_remote_*"),
randomFrom("index1", "*"),
randomBoolean()
)
);
return performRequestWithRemoteAccessUser(searchRequest);
}

private void indexTestData() throws Exception {
configureRemoteCluster();

// Fulfilling cluster
{
// Index some documents, so we can attempt to search them from the querying cluster
final Request bulkRequest = new Request("POST", "/_bulk?refresh=true");
bulkRequest.setJsonEntity(Strings.format("""
{ "index": { "_index": "index1" } }
{ "foo": "bar" }
{ "index": { "_index": "index2" } }
{ "bar": "foo" }
{ "index": { "_index": "prefixed_index" } }
{ "baz": "fee" }\n"""));
assertOK(performRequestAgainstFulfillingCluster(bulkRequest));
}

// Query cluster
{
// Index some documents, to use them in a mixed-cluster search
final var indexDocRequest = new Request("POST", "/local_index/_doc?refresh=true");
indexDocRequest.setJsonEntity("{\"local_foo\": \"local_bar\"}");
assertOK(client().performRequest(indexDocRequest));

// Create user role with privileges for remote and local indices
final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE);
putRoleRequest.setJsonEntity("""
{
"description": "role with privileges for remote and local indices",
"cluster": ["manage_own_api_key"],
"indices": [
{
"names": ["local_index"],
"privileges": ["read"]
}
],
"remote_indices": [
{
"names": ["index1", "not_found_index", "prefixed_index"],
"privileges": ["read", "read_cross_cluster"],
"clusters": ["my_remote_cluster"]
}
]
}""");
assertOK(adminClient().performRequest(putRoleRequest));
final var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER);
putUserRequest.setJsonEntity("""
{
"password": "x-pack-test-password",
"roles" : ["remote_search"]
}""");
assertOK(adminClient().performRequest(putUserRequest));
}
}

private void updateClusterSettingsFulfillingCluster(Settings settings) throws IOException {
final var request = newXContentRequest(HttpMethod.PUT, "/_cluster/settings", (builder, params) -> {
builder.startObject("persistent");
settings.toXContent(builder, params);
return builder.endObject();
});

performRequestWithAdminUser(fulfillingClusterClient, request);
}

private Response performRequestWithRemoteAccessUser(final Request request) throws IOException {
request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS)));
return client().performRequest(request);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -619,7 +619,7 @@ private static MockTransportService startTransport(
action,
SystemUser.crossClusterAccessSubjectInfo(TransportVersion.current(), nodeName)
)
).writeToContext(threadContext);
).writeToContext(threadContext, null);
connection.sendRequest(requestId, action, request, options);
}
});
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
-----BEGIN CERTIFICATE-----
MIIDSzCCAjOgAwIBAgIUPw9V/LIrB5Y+Krhqp1mXhK/BQDIwDQYJKoZIhvcNAQEL
BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
cmF0ZWQgQ0EwIBcNMjUwOTE4MDczMzQ2WhgPMjA1MzAyMDMwNzMzNDZaMDQxMjAw
BgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENB
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuQkSVLDqxWT83K+gfljq
WWL5KQxiN/7XZ8ug6e5b+kY0MZQnaAUWNaj5RFgSMDB+2N6EWJMk5cmDXK7NB/xq
gTbC/o3o7B9AZMTpu5Wbj8chRBCTTirRaIh79/VdLWDHriIgxGBLdMm/A2b3IW6H
YeUGUdOszEjytCjslrrktnIMIQHlQQ5o/fSPslCunsm7P+rDyf3GjapflKtpcrIV
cZKExaaCaqJtQYn66JCyr/ZFmjiTRPjPBcFx2SqAvkPXSK4SghvhKX69K+qDQWjF
rPx9BUWgLnE00bJ27CCSCRzZ3dTgcZ86ou/2mOJqqpeCMacJGWnn7Cu1P3+LcgjT
cQIDAQABo1MwUTAdBgNVHQ4EFgQUlL1P7M7/YCDULUeMUHPxDMtHT9swHwYDVR0j
BBgwFoAUlL1P7M7/YCDULUeMUHPxDMtHT9swDwYDVR0TAQH/BAUwAwEB/zANBgkq
hkiG9w0BAQsFAAOCAQEATCKw10zkCI21nuNppQZKFbHf/m3IZR9mZYYU0tKBSIy7
KoCTHZUTadbJuDzJ8eDRiqnUuXHUXNijykEphvfpckNDhb6ty5g707kET3EYDfkh
S1EKet2clM9DRqqcmFt3cyOmLJE3we7NjrNOuKNiwuXbGrqTTqNkqiiB3gWYOpSM
uwtCz1Syyl4y5sjocedkikqaeIKtl2htN3tEYd0BfLNVo5hN/syP8WDT6FdpCDpY
lZ2nqT622KDuusORCMTiC1qgUVR3RghPHy55Jq6Qq1+a1//E/Q9OfCs98JeUnoSp
W/q1hUVlSN0Edsn1T5LehGMjiH3UVszWvEThUqNHuA==
-----END CERTIFICATE-----
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
-----BEGIN CERTIFICATE-----
MIIDJTCCAg2gAwIBAgIVAIXaEONwJyih5d7KhnPYfqW5eNgKMA0GCSqGSIb3DQEB
CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
ZXJhdGVkIENBMCAXDTI1MDkxODA5NDQwMVoYDzIwNTMwMjAzMDk0NDAxWjATMREw
DwYDVQQDEwhpbnN0YW5jZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AJCOS/v9I34GaRD4BoMpIa/zhL1TBc1tqIDFKDm4Y/g4a+KUMJlaqHFRZkNI35LF
TkgMkRJUrWEunXJ2wf2A2s+7xWykSoa+nJ2qghwDgBtoSBWSm6I2Hi9n400Mmde/
xg912NzlfLJZH3la3/w3u7ENUY3GTNLeE5s5CpZAcOk+KQ/2/1Y7TgKPxyhbNtRA
2whWD862pnJypskQ9UGgB3Zq5h+2llQ2sB367pE77DyvXReKLHfCtA3lmTob6pLm
fK2cIBEJDwkFaAgrcWH5MwMkn+4v/Xw1PjAI4AVMOge+Rxt6waWxqQIJvOoyccXY
Vdvo8swUAjMPnR6E/5+bwykCAwEAAaNNMEswHQYDVR0OBBYEFH1XQX26JBIwvu95
xPhSCqOFrz9IMB8GA1UdIwQYMBaAFJS9T+zO/2Ag1C1HjFBz8QzLR0/bMAkGA1Ud
EwQCMAAwDQYJKoZIhvcNAQELBQADggEBAIF/LkOYm52Q+buBqGS380HWkNitTLG2
8qtICtXtLYd9673+c3RNIrW2CGFq3Z3TJ60FNvVT1z6NKiR8ZPUeqN+Avq5qN+dB
u9SPRFOrszlD6+2ZkNaZyRs2w6NQa6zBZWs0Zp3+ouu4fUEdsa/UmKud6njLaAGA
Rq8Sc7ckssykh1HKk8dOJt83GlvsBGXKALNv3vfHnMj+5XHC2NzZS5bn1IXWQE5z
0z5cHHD4NHiuGBnTl7MI8KzrF/Axwc2krsVO7WIQ/GpDVVwrCoKyvNm54GfpIAE/
ndH7bu9hGVM6swzpAdhQC/HK6Vc0NoGfoARXVRtxuEZmoq2amixHJJU=
-----END CERTIFICATE-----
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEogIBAAKCAQEAkI5L+/0jfgZpEPgGgykhr/OEvVMFzW2ogMUoObhj+Dhr4pQw
mVqocVFmQ0jfksVOSAyRElStYS6dcnbB/YDaz7vFbKRKhr6cnaqCHAOAG2hIFZKb
ojYeL2fjTQyZ17/GD3XY3OV8slkfeVrf/De7sQ1RjcZM0t4TmzkKlkBw6T4pD/b/
VjtOAo/HKFs21EDbCFYPzramcnKmyRD1QaAHdmrmH7aWVDawHfrukTvsPK9dF4os
d8K0DeWZOhvqkuZ8rZwgEQkPCQVoCCtxYfkzAySf7i/9fDU+MAjgBUw6B75HG3rB
pbGpAgm86jJxxdhV2+jyzBQCMw+dHoT/n5vDKQIDAQABAoIBAA1EKeAF4sB5mSHY
CUz3NOK/cAKqAGHSewDaVy8451/L2cbQ/8bLJaNEq6RoJzCCkAUXtiafA8xj6Uos
cPAxZ6Nh4aPvTfGgw6HKmKc2gQbC4r6sFkFkQw/pslgLXIEK1gPsNktLek6p1DQg
bWbpvH1qsf3XYYyGmfkIWprgbhxRl0Z/9dji+T7f23JvwMm+18e4RG4ic4Xb40gU
J6oYQ4ogev4f0VML+r2YUke+wx5NdwoM2L3uMUM7tf/T75AJBxQdQGiBSHB5zakX
z2/LtdWYxb7IDKKrFkdY99TWPIgtHtzZzdYr4/FUGx6cRs25F9okP2ucQ0U6UU1V
161bbsECgYEAwmGOGAdXaRXlOSfvENjf6sf9npz2KGlv8kEuBU9w0Gtxfx1q8Xx9
WUw4iI01LbYfBR/BWbCxOsn3SH5EQjQcReG0NyOo1G3a6S+NCfBKZms4DLWRA7fG
fF2ly9kvbvPz9089O7BWous9EqEnOEC+hkzMsb1lXYRDk8OQa2mlj1kCgYEAvmFR
EwcXuzfqJxHi6cEU+3bYrja3NOstWSBfvsulXc1G8tOcbqS4OGIRviQpA4+2JaK0
S4/YMiT3hUF+2lzZcnGSSrToKKeKrxNLUXoE4QLRjcVNmOIBSQdN+xbZVdFXeRCM
UnqBuw+gGmOhHeVicVWEbSjUce0FhIdHiQd/qFECgYAFU68VMX5Pvu3dNx7yEz9v
q7NjmWGVke4jcW3Vb2vkCk298gxwOb0lqVUTSOtgKVGITmp6DsGMnuRL9EnilpL/
x0OtDykdSTVqlocC8rbXP7D1iDRFKdAisF5Oy9Dk9YKGEIHZFOgK5u9xh0EP5ZZT
D9+8LziL64f+kKlwiCClYQKBgCB63+ccJatWPceOoKT6wQap3wvR3+3SVblH8a3O
dpcLR5h0C9NAnQFZkedbqfemlA/Vs2bU0rCzZ9s/MlI01xBUWf4O4TDWbK2z3/y1
kZGF9pR2XefAXzHDYkV9P3UJsx+/eAE2T13Hq6v05W8BTItDaMVq2tvY8UEMB2NU
eS4RAoGASViwa3uI8avQts1Jf4M4jIHTLzyqY8hfA+06BALe69nw4vWNsIFQlNot
IA2+276jZp0eoFtnleo+y9PD+5zZBWXYypGfw3XgVW2m7RI0x7Bic9zGEJlmgyZH
hUEbMlQjY1F6xv7+WWQCuiZf4ns7uHeseu8bczp0d/jT2ZQtMWE=
-----END RSA PRIVATE KEY-----
Loading