Skip to content

Commit b65f676

Browse files
quux00quux00
committed
Resolve/cluster should mark remotes as not connected when a security exception is thrown (#119793)
Fixes two bugs in _resolve/cluster. First, the code that detects older clusters versions and does a fallback to the _resolve/index endpoint was using an outdated string match for error detection. That has been adjusted. Second, upon security exceptions, the _resolve/cluster endpoint was marking the clusters as connected: true, under the assumption that all security exceptions related to cross cluster calls and remote index access were coming from the remote cluster, but that is not always the case. Some cross-cluster security violations can be detected on the local querying cluster after issuing the remoteClient.execute call but before the transport layer actually sends the request remotely. So we now mark the connected status as false for all ElasticsearchSecurityException cases. End user docs have been updated with this information.
1 parent b398448 commit b65f676

File tree

6 files changed

+55
-25
lines changed

6 files changed

+55
-25
lines changed

‎docs/changelog/119793.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 119793
2+
summary: Resolve/cluster should mark remotes as not connected when a security exception
3+
is thrown
4+
area: CCS
5+
type: bug
6+
issues: []

‎docs/reference/indices/resolve-cluster.asciidoc‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,19 @@ For each cluster in the index expression, information is returned about:
2828
3. whether there are any indices, aliases or data streams on that cluster that match
2929
the index expression
3030
4. whether the search is likely to have errors returned when you do the {ccs} (including any
31-
authorization errors if your user does not have permission to query the index)
32-
5. cluster version information, including the Elasticsearch server version
31+
authorization errors if your user does not have permission to query a remote cluster or
32+
the indices on that cluster)
33+
5. (in some cases) cluster version information, including the Elasticsearch server version
34+
35+
[TIP]
36+
====
37+
Whenever a security exception is returned for a remote cluster, that remote
38+
will always be marked as connected=false in the response, since your user does not have
39+
permissions to access that cluster (or perhaps the remote index) you are querying.
40+
Once the proper security permissions are obtained, then you can rely on the `connected` field
41+
in the response to determine whether the remote cluster is available and ready for querying.
42+
====
43+
3344

3445
////
3546
[source,console]

‎server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@
99

1010
package org.elasticsearch.action.admin.indices.resolve;
1111

12+
import org.elasticsearch.TransportVersion;
1213
import org.elasticsearch.TransportVersions;
1314
import org.elasticsearch.action.ActionRequest;
1415
import org.elasticsearch.action.ActionRequestValidationException;
1516
import org.elasticsearch.action.IndicesRequest;
1617
import org.elasticsearch.action.ValidateActions;
1718
import org.elasticsearch.action.support.IndicesOptions;
19+
import org.elasticsearch.common.Strings;
1820
import org.elasticsearch.common.io.stream.StreamInput;
1921
import org.elasticsearch.common.io.stream.StreamOutput;
2022
import org.elasticsearch.tasks.CancellableTask;
@@ -30,6 +32,7 @@
3032
public class ResolveClusterActionRequest extends ActionRequest implements IndicesRequest.Replaceable {
3133

3234
public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpen();
35+
public static final String TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX = "ResolveClusterAction requires at least version";
3336

3437
private String[] names;
3538
/*
@@ -65,12 +68,7 @@ public ResolveClusterActionRequest(String[] names, IndicesOptions indicesOptions
6568
public ResolveClusterActionRequest(StreamInput in) throws IOException {
6669
super(in);
6770
if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
68-
throw new UnsupportedOperationException(
69-
"ResolveClusterAction requires at least version "
70-
+ TransportVersions.V_8_13_0.toReleaseVersion()
71-
+ " but was "
72-
+ in.getTransportVersion().toReleaseVersion()
73-
);
71+
throw new UnsupportedOperationException(createVersionErrorMessage(in.getTransportVersion()));
7472
}
7573
this.names = in.readStringArray();
7674
this.indicesOptions = IndicesOptions.readIndicesOptions(in);
@@ -81,17 +79,21 @@ public ResolveClusterActionRequest(StreamInput in) throws IOException {
8179
public void writeTo(StreamOutput out) throws IOException {
8280
super.writeTo(out);
8381
if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
84-
throw new UnsupportedOperationException(
85-
"ResolveClusterAction requires at least version "
86-
+ TransportVersions.V_8_13_0.toReleaseVersion()
87-
+ " but was "
88-
+ out.getTransportVersion().toReleaseVersion()
89-
);
82+
throw new UnsupportedOperationException(createVersionErrorMessage(out.getTransportVersion()));
9083
}
9184
out.writeStringArray(names);
9285
indicesOptions.writeIndicesOptions(out);
9386
}
9487

88+
private String createVersionErrorMessage(TransportVersion versionFound) {
89+
return Strings.format(
90+
"%s %s but was %s",
91+
TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX,
92+
TransportVersions.V_8_13_0.toReleaseVersion(),
93+
versionFound.toReleaseVersion()
94+
);
95+
}
96+
9597
@Override
9698
public ActionRequestValidationException validate() {
9799
ActionRequestValidationException validationException = null;

‎server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@
5151
public class TransportResolveClusterAction extends HandledTransportAction<ResolveClusterActionRequest, ResolveClusterActionResponse> {
5252

5353
private static final Logger logger = LogManager.getLogger(TransportResolveClusterAction.class);
54-
private static final String TRANSPORT_VERSION_ERROR_MESSAGE = "ResolveClusterAction requires at least Transport Version";
5554

5655
public static final String NAME = "indices:admin/resolve/cluster";
5756
public static final ActionType<ResolveClusterActionResponse> TYPE = new ActionType<>(NAME);
@@ -175,7 +174,13 @@ public void onFailure(Exception failure) {
175174
failure,
176175
ElasticsearchSecurityException.class
177176
) instanceof ElasticsearchSecurityException ese) {
178-
clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(true, skipUnavailable, ese.getMessage()));
177+
/*
178+
* some ElasticsearchSecurityExceptions come from the local cluster security interceptor after you've
179+
* issued the client.execute call but before any call went to the remote cluster, so with an
180+
* ElasticsearchSecurityException you can't tell whether the remote cluster is available or not, so mark
181+
* it as connected=false
182+
*/
183+
clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(false, skipUnavailable, ese.getMessage()));
179184
} else if (ExceptionsHelper.unwrap(failure, IndexNotFoundException.class) instanceof IndexNotFoundException infe) {
180185
clusterInfoMap.put(clusterAlias, new ResolveClusterInfo(true, skipUnavailable, infe.getMessage()));
181186
} else {
@@ -184,7 +189,7 @@ public void onFailure(Exception failure) {
184189
// this error at the Transport layer BEFORE it sends the request to the remote cluster, since there
185190
// are version guards on the Writeables for this Action, namely ResolveClusterActionRequest.writeTo
186191
if (cause instanceof UnsupportedOperationException
187-
&& cause.getMessage().contains(TRANSPORT_VERSION_ERROR_MESSAGE)) {
192+
&& cause.getMessage().contains(ResolveClusterActionRequest.TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX)) {
188193
// Since this cluster does not have _resolve/cluster, we call the _resolve/index
189194
// endpoint to fill in the matching_indices field of the response for that cluster
190195
ResolveIndexAction.Request resolveIndexRequest = new ResolveIndexAction.Request(

‎x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS1ResolveClusterIT.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ public void testResolveClusterUnderRCS1() throws Exception {
9393
assertLocalMatching(responseMap);
9494

9595
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
96-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
96+
// with security exceptions, the remote should be marked as connected=false, since you can't tell whether a security
97+
// exception comes from the local cluster (intercepted) or the remote
98+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
9799
assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
98100

99101
// TEST CASE 2: Query cluster -> add user role and user on remote cluster and try resolve again
@@ -171,7 +173,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
171173
Map<String, Object> responseMap = responseAsMap(response);
172174
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
173175
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
174-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
176+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
175177
assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
176178
assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
177179
}
@@ -183,7 +185,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
183185
Map<String, Object> responseMap = responseAsMap(response);
184186
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
185187
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
186-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
188+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
187189
assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
188190
assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [doesnotexist]"));
189191
}
@@ -195,6 +197,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
195197
Map<String, Object> responseMap = responseAsMap(response);
196198
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
197199
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
200+
// with IndexNotFoundExceptions, we know that error came from the remote cluster, so we can mark the remote as connected=true
198201
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
199202
assertThat((String) remoteClusterResponse.get("error"), containsString("no such index [index99]"));
200203
}
@@ -210,7 +213,7 @@ public void testResolveClusterUnderRCS1() throws Exception {
210213
Map<String, Object> responseMap = responseAsMap(response);
211214
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
212215
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
213-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
216+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
214217
assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]"));
215218
assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
216219
}

‎x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,9 @@ public void testResolveCluster() throws Exception {
180180
assertLocalMatching(responseMap);
181181

182182
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
183-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
183+
// with security exceptions, the remote should be marked as connected=false, since you can't tell whether a security
184+
// exception comes from the local cluster (intercepted) or the remote
185+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
184186
assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
185187
assertThat(
186188
(String) remoteClusterResponse.get("error"),
@@ -261,7 +263,7 @@ public void testResolveCluster() throws Exception {
261263
Map<String, Object> responseMap = responseAsMap(response);
262264
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
263265
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
264-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
266+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
265267
assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
266268
assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
267269
}
@@ -273,7 +275,7 @@ public void testResolveCluster() throws Exception {
273275
Map<String, Object> responseMap = responseAsMap(response);
274276
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
275277
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
276-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
278+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
277279
assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
278280
assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [doesnotexist]"));
279281
}
@@ -285,6 +287,7 @@ public void testResolveCluster() throws Exception {
285287
Map<String, Object> responseMap = responseAsMap(response);
286288
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
287289
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
290+
// with IndexNotFoundExceptions, we know that error came from the remote cluster, so we can mark the remote as connected=true
288291
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
289292
assertThat((Boolean) remoteClusterResponse.get("skip_unavailable"), equalTo(false));
290293
assertThat((String) remoteClusterResponse.get("error"), containsString("no such index [index99]"));
@@ -301,7 +304,7 @@ public void testResolveCluster() throws Exception {
301304
Map<String, Object> responseMap = responseAsMap(response);
302305
assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue());
303306
Map<String, ?> remoteClusterResponse = (Map<String, ?>) responseMap.get("my_remote_cluster");
304-
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true));
307+
assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false));
305308
assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user"));
306309
assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]"));
307310
}

0 commit comments

Comments
 (0)