Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
2bc0e99
Prevent usage of :: selectors for remote cluster requests
slobodanadamovic Mar 19, 2025
fc2271c
fail only if ::failures selector is used
slobodanadamovic Mar 19, 2025
ba976a9
cleanup and extend existing rest IT
slobodanadamovic Mar 19, 2025
f6c96e2
Merge branch 'main' into sa-prevent-using-selectors-for-ccs-ccr
slobodanadamovic Mar 19, 2025
5bc46b5
test ccs with rcs1
slobodanadamovic Mar 20, 2025
e92ffde
nit
slobodanadamovic Mar 20, 2025
29de669
remove remote_indices - not relevant for RCS1 test case
slobodanadamovic Mar 20, 2025
eaf9a01
test CCS with RCS2
slobodanadamovic Mar 20, 2025
41da2e6
Merge branch 'main' of github.com:elastic/elasticsearch into sa-preve…
slobodanadamovic Mar 20, 2025
4e366ed
cleanup tests, handle edge cases with ccs_minimize_roundtrips
slobodanadamovic Mar 21, 2025
542aa54
Merge branch 'main' of github.com:elastic/elasticsearch into sa-preve…
slobodanadamovic Mar 21, 2025
816d919
more test users
slobodanadamovic Mar 21, 2025
7dc89d4
fix assertion
slobodanadamovic Mar 21, 2025
26e8ae3
test direct access to backing failure index for other users
slobodanadamovic Mar 21, 2025
9c3d0d7
fix assertion
slobodanadamovic Mar 21, 2025
e8a4b96
fix another assertion
slobodanadamovic Mar 21, 2025
6c8eee8
Merge branch 'main' of github.com:elastic/elasticsearch into sa-preve…
slobodanadamovic Mar 25, 2025
6d5293f
bring back selector validation inline
slobodanadamovic Mar 25, 2025
fd1a0e5
refactor to reuse selector validation
slobodanadamovic Mar 25, 2025
c88a35f
Merge branch 'main' of github.com:elastic/elasticsearch into sa-preve…
slobodanadamovic Mar 25, 2025
1bd03a6
more tests
slobodanadamovic Mar 25, 2025
0dd4053
more tests 2
slobodanadamovic Mar 25, 2025
f085835
more test coverage
slobodanadamovic Mar 25, 2025
446f8b6
Merge branch 'main' of github.com:elastic/elasticsearch into sa-preve…
slobodanadamovic Mar 25, 2025
62664ac
[CI] Auto commit changes from spotless
Mar 25, 2025
0fe3ec5
prevent using ::data selector as well
slobodanadamovic Mar 26, 2025
ae8cc1a
Merge branch 'main' into sa-prevent-using-selectors-for-ccs-ccr
slobodanadamovic Mar 26, 2025
4baa39a
consolidate error messages and exceptions thrown
slobodanadamovic Mar 26, 2025
5811200
fix failing test
slobodanadamovic Mar 26, 2025
bc0a5b5
Merge branch 'main' into sa-prevent-using-selectors-for-ccs-ccr
slobodanadamovic Mar 26, 2025
7d1dd3e
fix failing test ::data is not allowed
slobodanadamovic Mar 27, 2025
5056909
Merge branch 'main' of github.com:elastic/elasticsearch into sa-preve…
slobodanadamovic Mar 27, 2025
321d5c7
Merge branch 'main' into sa-prevent-using-selectors-for-ccs-ccr
slobodanadamovic Mar 27, 2025
148567d
Merge branch 'main' into sa-prevent-using-selectors-for-ccs-ccr
slobodanadamovic Mar 27, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2373,6 +2373,7 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
}
String expressionBase = expression.substring(0, lastDoubleColon);
ensureNoMoreSelectorSeparators(expressionBase, expression);
ensureNoCrossClusterExpressionWithFailuresSelector(expressionBase, selector, expression);
return bindFunction.apply(expressionBase, suffix);
}
// Otherwise accept the default
Expand Down Expand Up @@ -2407,6 +2408,29 @@ private static void ensureNoMoreSelectorSeparators(String remainingExpression, S
);
}
}

/**
* Checks the expression for cross-cluster syntax and throws an exception if it is combined with ::failures selector.
* @throws IllegalArgumentException if cross-cluster syntax is detected after parsing the selector expression
*/
private static void ensureNoCrossClusterExpressionWithFailuresSelector(
String expressionWithoutSelector,
IndexComponentSelector selector,
String originalExpression
) {
if (selector == IndexComponentSelector.FAILURES) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we simply fully bar any selector usage with cross cluster search? If the only supported option is ::data, and no selector is the same as that, perhaps we should just disallow the syntax in case the suffix of ::data causes issues further down the line.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I originally suggested only restricting ::failures but on second thought restricting ::data makes sense -- I guess it makes things simpler to just not allow any selectors at all.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for raising this. If there is no real need to support ::data it would make this more consistent and simpler. I'll make the change.

if (RemoteClusterAware.isRemoteIndexName(expressionWithoutSelector)) {
throw new IllegalArgumentException(
"Invalid usage of ["
+ SELECTOR_SEPARATOR
+ selector.getKey()
+ "] selector in ["
+ originalExpression
+ "], failures selector is not supported with cross-cluster expressions"
);
}
}
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.elasticsearch.action.support.IndexComponentSelector.FAILURES;
import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.Context;
import static org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
Expand Down Expand Up @@ -72,16 +73,36 @@ public void testResolveExpression() {
// === Corner Cases
// Empty index name is not necessarily disallowed, but will be filtered out in the next steps of resolution
assertThat(resolve(selectorsAllowed, "::data"), equalTo(new ResolvedExpression("", DATA)));
assertThat(resolve(selectorsAllowed, "::failures"), equalTo(new ResolvedExpression("", FAILURES)));
// Remote cluster syntax is respected, even if code higher up the call stack is likely to already have handled it already
assertThat(resolve(selectorsAllowed, "cluster:index::data"), equalTo(new ResolvedExpression("cluster:index", DATA)));
// CCS with an empty index name is not necessarily disallowed, though other code in the resolution logic will likely throw
assertThat(resolve(selectorsAllowed, "cluster:::data"), equalTo(new ResolvedExpression("cluster:", DATA)));
// Same for empty cluster and index names
assertThat(resolve(selectorsAllowed, ":::data"), equalTo(new ResolvedExpression(":", DATA)));
assertThat(resolve(selectorsAllowed, ":::failures"), equalTo(new ResolvedExpression(":", FAILURES)));
// Any more prefix colon characters will trigger the multiple separators error logic
expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "::::data"));
expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "::::failures"));
expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, ":::::failures"));
// Suffix case is not supported because there is no component named with the empty string
expectThrows(InvalidIndexNameException.class, () -> resolve(selectorsAllowed, "index::"));

// remote cluster syntax is not allowed with ::failures selector
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "cluster:index::failures");
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(noSelectors, "cluster:index::failures");
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "cluster-*:index::failures");
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "cluster-*:index-*::failures");
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "cluster-*:*::failures");
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "*:index-*::failures");
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "*:*::failures");
// even with an empty index name
assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(selectorsAllowed, "cluster:::failures");
}

public void assertFailuresSelectorNotSupportedWithRemoteClusterExpressions(Context context, String expression) {
var e = expectThrows(IllegalArgumentException.class, () -> resolve(context, expression));
assertThat(e.getMessage(), containsString("failures selector is not supported with cross-cluster expressions"));
}

public void testResolveMatchAllToSelectors() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.remotecluster;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchResponseUtils;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;

abstract class AbstractRemoteClusterSecurityFailureStoreRestIT extends AbstractRemoteClusterSecurityTestCase {

protected void assertSearchResponseContainsIndices(Response response, String... expectedIndices) throws IOException {
assertOK(response);
final SearchResponse searchResponse = SearchResponseUtils.parseSearchResponse(responseAsParser(response));
try {
final List<String> actualIndices = Arrays.stream(searchResponse.getHits().getHits())
.map(SearchHit::getIndex)
.collect(Collectors.toList());
assertThat(actualIndices, containsInAnyOrder(expectedIndices));
} finally {
searchResponse.decRef();
}
}

protected void setupTestDataStreamOnFulfillingCluster() throws IOException {
// Create data stream and index some documents
final Request createComponentTemplate = new Request("PUT", "/_component_template/component1");
createComponentTemplate.setJsonEntity("""
{
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"age": {
"type": "integer"
},
"email": {
"type": "keyword"
},
"name": {
"type": "text"
}
}
},
"data_stream_options": {
"failure_store": {
"enabled": true
}
}
}
}""");
assertOK(performRequestAgainstFulfillingCluster(createComponentTemplate));

final Request createTemplate = new Request("PUT", "/_index_template/template1");
createTemplate.setJsonEntity("""
{
"index_patterns": ["test*"],
"data_stream": {},
"priority": 500,
"composed_of": ["component1"]
}""");
assertOK(performRequestAgainstFulfillingCluster(createTemplate));

final Request createDoc1 = new Request("PUT", "/test1/_doc/1?refresh=true&op_type=create");
createDoc1.setJsonEntity("""
{
"@timestamp": 1,
"age" : 1,
"name" : "jack",
"email" : "[email protected]"
}""");
assertOK(performRequestAgainstFulfillingCluster(createDoc1));

final Request createDoc2 = new Request("PUT", "/test1/_doc/2?refresh=true&op_type=create");
createDoc2.setJsonEntity("""
{
"@timestamp": 2,
"age" : "this should be an int",
"name" : "jack",
"email" : "[email protected]"
}""");
assertOK(performRequestAgainstFulfillingCluster(createDoc2));
}

protected Response performRequestWithRemoteSearchUser(final Request request) throws IOException {
request.setOptions(
RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(REMOTE_SEARCH_USER, PASS))
);
return client().performRequest(request);
}

protected Response performRequestWithUser(final String user, final Request request) throws IOException {
request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(user, PASS)));
return client().performRequest(request);
}

@SuppressWarnings("unchecked")
protected Tuple<List<String>, List<String>> getDataAndFailureIndices(String dataStreamName) throws IOException {
Request dataStream = new Request("GET", "/_data_stream/" + dataStreamName);
Response response = performRequestAgainstFulfillingCluster(dataStream);
Map<String, Object> dataStreams = entityAsMap(response);
assertEquals(Collections.singletonList("test1"), XContentMapValues.extractValue("data_streams.name", dataStreams));
List<String> dataIndexNames = (List<String>) XContentMapValues.extractValue("data_streams.indices.index_name", dataStreams);
List<String> failureIndexNames = (List<String>) XContentMapValues.extractValue(
"data_streams.failure_store.indices.index_name",
dataStreams
);
return new Tuple<>(dataIndexNames, failureIndexNames);
}

protected Tuple<String, String> getSingleDataAndFailureIndices(String dataStreamName) throws IOException {
Tuple<List<String>, List<String>> indices = getDataAndFailureIndices(dataStreamName);
assertThat(indices.v1().size(), equalTo(1));
assertThat(indices.v2().size(), equalTo(1));
return new Tuple<>(indices.v1().get(0), indices.v2().get(0));
}

}
Loading