Skip to content

Commit 72a968d

Browse files
n1v0lgn1v0lg
committed
Add CCS tests for ESQL with DLS
1 parent 5996772 commit 72a968d

File tree

3 files changed

+380
-0
lines changed

3 files changed

+380
-0
lines changed

‎x-pack/plugin/security/qa/multi-cluster/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ dependencies {
2424
clusterModules project(':x-pack:plugin:enrich')
2525
clusterModules project(':x-pack:plugin:autoscaling')
2626
clusterModules project(':x-pack:plugin:ml')
27+
clusterModules project(xpackModule('ilm'))
2728
clusterModules(project(":modules:ingest-common"))
2829
}
2930

‎x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java‎

Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.remotecluster;
9+
10+
import org.elasticsearch.Build;
11+
import org.elasticsearch.client.Request;
12+
import org.elasticsearch.client.RequestOptions;
13+
import org.elasticsearch.client.Response;
14+
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.test.MapMatcher;
16+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
17+
import org.elasticsearch.test.cluster.util.resource.Resource;
18+
import org.elasticsearch.test.junit.RunnableTestRuleAdapter;
19+
import org.elasticsearch.xcontent.XContentBuilder;
20+
import org.elasticsearch.xcontent.json.JsonXContent;
21+
import org.junit.ClassRule;
22+
import org.junit.rules.RuleChain;
23+
import org.junit.rules.TestRule;
24+
25+
import java.io.IOException;
26+
import java.util.Locale;
27+
import java.util.Map;
28+
import java.util.concurrent.atomic.AtomicBoolean;
29+
import java.util.concurrent.atomic.AtomicReference;
30+
31+
import static org.elasticsearch.test.ListMatcher.matchesList;
32+
import static org.elasticsearch.test.MapMatcher.assertMap;
33+
import static org.elasticsearch.test.MapMatcher.matchesMap;
34+
35+
public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteClusterSecurityTestCase {
36+
private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();
37+
private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean();
38+
private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean();
39+
private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean();
40+
41+
static {
42+
fulfillingCluster = ElasticsearchCluster.local()
43+
.name("fulfilling-cluster")
44+
.nodes(3)
45+
.module("x-pack-autoscaling")
46+
.module("x-pack-esql")
47+
.module("x-pack-enrich")
48+
.module("x-pack-ml")
49+
.module("x-pack-ilm")
50+
.module("ingest-common")
51+
.apply(commonClusterConfig)
52+
.setting("remote_cluster.port", "0")
53+
.setting("xpack.ml.enabled", "false")
54+
.setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
55+
.setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
56+
.setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
57+
.setting("xpack.security.authc.token.enabled", "true")
58+
.keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
59+
.node(0, spec -> spec.setting("remote_cluster_server.enabled", "true"))
60+
.node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get())))
61+
.node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get())))
62+
.build();
63+
64+
queryCluster = ElasticsearchCluster.local()
65+
.name("query-cluster")
66+
.module("x-pack-autoscaling")
67+
.module("x-pack-esql")
68+
.module("x-pack-enrich")
69+
.module("x-pack-ml")
70+
.module("x-pack-ilm")
71+
.module("ingest-common")
72+
.apply(commonClusterConfig)
73+
.setting("xpack.ml.enabled", "false")
74+
.setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get()))
75+
.setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
76+
.setting("xpack.security.authc.token.enabled", "true")
77+
.keystore("cluster.remote.my_remote_cluster.credentials", () -> {
78+
if (API_KEY_MAP_REF.get() == null) {
79+
final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
80+
{
81+
"search": [
82+
{
83+
"names": ["logs-*", "alias-*"]
84+
}
85+
]
86+
}""");
87+
API_KEY_MAP_REF.set(apiKeyMap);
88+
}
89+
return (String) API_KEY_MAP_REF.get().get("encoded");
90+
})
91+
.rolesFile(Resource.fromClasspath("roles.yml"))
92+
.user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false)
93+
.user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false)
94+
.user("logs_foo_after_2021", "x-pack-test-password", "logs_foo_after_2021", false)
95+
.user("logs_foo_after_2021_pattern", "x-pack-test-password", "logs_foo_after_2021_pattern", false)
96+
.user("logs_foo_after_2021_alias", "x-pack-test-password", "logs_foo_after_2021_alias", false)
97+
.build();
98+
}
99+
100+
@ClassRule
101+
// Use a RuleChain to ensure that fulfilling cluster is started before query cluster
102+
// `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters
103+
// We set it here, since randomization methods are not available in the static initialize context above
104+
public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> {
105+
SSL_ENABLED_REF.set(usually());
106+
NODE1_RCS_SERVER_ENABLED.set(randomBoolean());
107+
NODE2_RCS_SERVER_ENABLED.set(randomBoolean());
108+
})).around(fulfillingCluster).around(queryCluster);
109+
110+
public void testDataStreamWithDls() throws Exception {
111+
configureRemoteCluster();
112+
createDataStream();
113+
MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2)));
114+
MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1)));
115+
assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), twoResults);
116+
assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), oneResult);
117+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), oneResult);
118+
assertMap(
119+
entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")),
120+
oneResult
121+
);
122+
assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), twoResults);
123+
assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult);
124+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult);
125+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult);
126+
127+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)")), oneResult);
128+
assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), oneResult);
129+
}
130+
131+
protected Response runESQLCommand(String user, String command) throws IOException {
132+
if (command.toLowerCase(Locale.ROOT).contains("limit") == false) {
133+
// add a (high) limit to avoid warnings on default limit
134+
command += " | limit 10000000";
135+
}
136+
XContentBuilder json = JsonXContent.contentBuilder();
137+
json.startObject();
138+
json.field("query", command);
139+
addRandomPragmas(json);
140+
json.endObject();
141+
Request request = new Request("POST", "_query");
142+
request.setJsonEntity(org.elasticsearch.common.Strings.toString(json));
143+
request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user));
144+
request.addParameter("error_trace", "true");
145+
return adminClient().performRequest(request);
146+
}
147+
148+
static void addRandomPragmas(XContentBuilder builder) throws IOException {
149+
if (Build.current().isSnapshot()) {
150+
Settings pragmas = randomPragmas();
151+
if (pragmas != Settings.EMPTY) {
152+
builder.startObject("pragma");
153+
builder.value(pragmas);
154+
builder.endObject();
155+
}
156+
}
157+
}
158+
159+
static Settings randomPragmas() {
160+
Settings.Builder settings = Settings.builder();
161+
if (randomBoolean()) {
162+
settings.put("page_size", between(1, 5));
163+
}
164+
if (randomBoolean()) {
165+
settings.put("exchange_buffer_size", between(1, 2));
166+
}
167+
if (randomBoolean()) {
168+
settings.put("data_partitioning", randomFrom("shard", "segment", "doc"));
169+
}
170+
if (randomBoolean()) {
171+
settings.put("enrich_max_workers", between(1, 5));
172+
}
173+
if (randomBoolean()) {
174+
settings.put("node_level_reduction", randomBoolean());
175+
}
176+
return settings.build();
177+
}
178+
179+
private void createDataStream() throws IOException {
180+
createDataStreamPolicy();
181+
createDataStreamComponentTemplate();
182+
createDataStreamIndexTemplate();
183+
createDataStreamDocuments();
184+
createDataStreamAlias();
185+
}
186+
187+
private void createDataStreamPolicy() throws IOException {
188+
Request request = new Request("PUT", "_ilm/policy/my-lifecycle-policy");
189+
request.setJsonEntity("""
190+
{
191+
"policy": {
192+
"phases": {
193+
"hot": {
194+
"actions": {
195+
"rollover": {
196+
"max_primary_shard_size": "50gb"
197+
}
198+
}
199+
},
200+
"delete": {
201+
"min_age": "735d",
202+
"actions": {
203+
"delete": {}
204+
}
205+
}
206+
}
207+
}
208+
}""");
209+
210+
performRequestAgainstFulfillingCluster(request);
211+
}
212+
213+
private void createDataStreamComponentTemplate() throws IOException {
214+
Request request = new Request("PUT", "_component_template/my-template");
215+
request.setJsonEntity("""
216+
{
217+
"template": {
218+
"settings": {
219+
"index.lifecycle.name": "my-lifecycle-policy"
220+
},
221+
"mappings": {
222+
"properties": {
223+
"@timestamp": {
224+
"type": "date",
225+
"format": "date_optional_time||epoch_millis"
226+
},
227+
"data_stream": {
228+
"properties": {
229+
"namespace": {"type": "keyword"}
230+
}
231+
}
232+
}
233+
}
234+
}
235+
}""");
236+
performRequestAgainstFulfillingCluster(request);
237+
}
238+
239+
private void createDataStreamIndexTemplate() throws IOException {
240+
Request request = new Request("PUT", "_index_template/my-index-template");
241+
request.setJsonEntity("""
242+
{
243+
"index_patterns": ["logs-*"],
244+
"data_stream": {},
245+
"composed_of": ["my-template"],
246+
"priority": 500
247+
}""");
248+
performRequestAgainstFulfillingCluster(request);
249+
}
250+
251+
private void createDataStreamDocuments() throws IOException {
252+
Request request = new Request("POST", "logs-foo/_bulk");
253+
request.addParameter("refresh", "");
254+
request.setJsonEntity("""
255+
{ "create" : {} }
256+
{ "@timestamp": "2099-05-06T16:21:15.000Z", "data_stream": {"namespace": "16"} }
257+
{ "create" : {} }
258+
{ "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17"} }
259+
""");
260+
assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false));
261+
}
262+
263+
private void createDataStreamAlias() throws IOException {
264+
Request request = new Request("PUT", "_alias");
265+
request.setJsonEntity("""
266+
{
267+
"actions": [
268+
{
269+
"add": {
270+
"index": "logs-foo",
271+
"alias": "alias-foo"
272+
}
273+
}
274+
]
275+
}""");
276+
assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false));
277+
}
278+
279+
record ExpectedCluster(String clusterAlias, String indexExpression, String status, Integer totalShards) {}
280+
}

‎x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,102 @@ ccr_user_role:
4141

4242
manage_role:
4343
cluster: [ 'manage' ]
44+
45+
logs_foo_all:
46+
cluster: []
47+
indices:
48+
- names: [ 'logs-foo' ]
49+
privileges: [ 'read' ]
50+
remote_indices:
51+
- names: [ 'logs-foo' ]
52+
clusters: [ '*' ]
53+
privileges: [ 'read' ]
54+
55+
logs_foo_16_only:
56+
cluster: []
57+
indices:
58+
- names: [ 'logs-foo' ]
59+
privileges: [ 'read' ]
60+
query: |
61+
{
62+
"term": {
63+
"data_stream.namespace": "16"
64+
}
65+
}
66+
remote_indices:
67+
- names: [ 'logs-foo' ]
68+
clusters: [ '*' ]
69+
privileges: [ 'read' ]
70+
query: |
71+
{
72+
"term": {
73+
"data_stream.namespace": "16"
74+
}
75+
}
76+
77+
logs_foo_after_2021:
78+
cluster: []
79+
indices:
80+
- names: [ 'logs-foo' ]
81+
privileges: [ 'read' ]
82+
query: |
83+
{
84+
"range": {
85+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
86+
}
87+
}
88+
remote_indices:
89+
- names: [ 'logs-foo' ]
90+
clusters: [ '*' ]
91+
privileges: [ 'read' ]
92+
query: |
93+
{
94+
"range": {
95+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
96+
}
97+
}
98+
99+
logs_foo_after_2021_pattern:
100+
cluster: []
101+
indices:
102+
- names: [ 'logs-*' ]
103+
privileges: [ 'read' ]
104+
query: |
105+
{
106+
"range": {
107+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
108+
}
109+
}
110+
remote_indices:
111+
- names: [ 'logs-*' ]
112+
clusters: [ '*' ]
113+
privileges: [ 'read' ]
114+
query: |
115+
{
116+
"range": {
117+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
118+
}
119+
}
120+
121+
logs_foo_after_2021_alias:
122+
cluster: []
123+
indices:
124+
- names: [ 'alias-foo' ]
125+
privileges: [ 'read' ]
126+
query: |
127+
{
128+
"range": {
129+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
130+
}
131+
}
132+
remote_indices:
133+
- names: [ 'alias-foo' ]
134+
clusters: [ '*' ]
135+
privileges: [ 'read' ]
136+
query: |
137+
{
138+
"range": {
139+
"@timestamp": {"gte": "2021-01-01T00:00:00"}
140+
}
141+
}
142+

0 commit comments

Comments
 (0)