From 72a968dd93c820d5bd5a5158507f180a26287dd2 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 11 Dec 2024 10:24:35 +0100 Subject: [PATCH 1/5] Add CCS tests for ESQL with DLS --- .../security/qa/multi-cluster/build.gradle | 1 + ...RemoteClusterSecurityDataStreamEsqlIT.java | 280 ++++++++++++++++++ .../src/javaRestTest/resources/roles.yml | 99 +++++++ 3 files changed, 380 insertions(+) create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java diff --git a/x-pack/plugin/security/qa/multi-cluster/build.gradle b/x-pack/plugin/security/qa/multi-cluster/build.gradle index 5b682cfdccade..f4eee4ef46c02 100644 --- a/x-pack/plugin/security/qa/multi-cluster/build.gradle +++ b/x-pack/plugin/security/qa/multi-cluster/build.gradle @@ -24,6 +24,7 @@ dependencies { clusterModules project(':x-pack:plugin:enrich') clusterModules project(':x-pack:plugin:autoscaling') clusterModules project(':x-pack:plugin:ml') + clusterModules project(xpackModule('ilm')) clusterModules(project(":modules:ingest-common")) } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java new file mode 100644 index 0000000000000..abfa1a6f1fb78 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java @@ -0,0 +1,280 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.Build; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; + +public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteClusterSecurityTestCase { + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(3) + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get()))) + .node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get()))) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["logs-*", "alias-*"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false) + .user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false) + .user("logs_foo_after_2021", "x-pack-test-password", "logs_foo_after_2021", false) + .user("logs_foo_after_2021_pattern", "x-pack-test-password", "logs_foo_after_2021_pattern", false) + .user("logs_foo_after_2021_alias", "x-pack-test-password", "logs_foo_after_2021_alias", false) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + // `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters + // We set it here, since randomization methods are not available in the static initialize context above + public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> { + SSL_ENABLED_REF.set(usually()); + NODE1_RCS_SERVER_ENABLED.set(randomBoolean()); + NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); + })).around(fulfillingCluster).around(queryCluster); + + public void testDataStreamWithDls() throws Exception { + configureRemoteCluster(); + createDataStream(); + MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); + MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); + assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), twoResults); + assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), oneResult); + assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), oneResult); + assertMap( + entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), twoResults); + assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult); + assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult); + assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult); + + assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)")), oneResult); + assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), oneResult); + } + + protected Response runESQLCommand(String user, String command) throws IOException { + if (command.toLowerCase(Locale.ROOT).contains("limit") == false) { + // add a (high) limit to avoid warnings on default limit + command += " | limit 10000000"; + } + XContentBuilder json = JsonXContent.contentBuilder(); + json.startObject(); + json.field("query", command); + addRandomPragmas(json); + json.endObject(); + Request request = new Request("POST", "_query"); + request.setJsonEntity(org.elasticsearch.common.Strings.toString(json)); + request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user)); + request.addParameter("error_trace", "true"); + return adminClient().performRequest(request); + } + + static void addRandomPragmas(XContentBuilder builder) throws IOException { + if (Build.current().isSnapshot()) { + Settings pragmas = randomPragmas(); + if (pragmas != Settings.EMPTY) { + builder.startObject("pragma"); + builder.value(pragmas); + builder.endObject(); + } + } + } + + static Settings randomPragmas() { + Settings.Builder settings = Settings.builder(); + if (randomBoolean()) { + settings.put("page_size", between(1, 5)); + } + if (randomBoolean()) { + settings.put("exchange_buffer_size", between(1, 2)); + } + if (randomBoolean()) { + settings.put("data_partitioning", randomFrom("shard", "segment", "doc")); + } + if (randomBoolean()) { + settings.put("enrich_max_workers", between(1, 5)); + } + if (randomBoolean()) { + settings.put("node_level_reduction", randomBoolean()); + } + return settings.build(); + } + + private void createDataStream() throws IOException { + createDataStreamPolicy(); + createDataStreamComponentTemplate(); + createDataStreamIndexTemplate(); + createDataStreamDocuments(); + createDataStreamAlias(); + } + + private void createDataStreamPolicy() throws IOException { + Request request = new Request("PUT", "_ilm/policy/my-lifecycle-policy"); + request.setJsonEntity(""" + { + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "50gb" + } + } + }, + "delete": { + "min_age": "735d", + "actions": { + "delete": {} + } + } + } + } + }"""); + + performRequestAgainstFulfillingCluster(request); + } + + private void createDataStreamComponentTemplate() throws IOException { + Request request = new Request("PUT", "_component_template/my-template"); + request.setJsonEntity(""" + { + "template": { + "settings": { + "index.lifecycle.name": "my-lifecycle-policy" + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "data_stream": { + "properties": { + "namespace": {"type": "keyword"} + } + } + } + } + } + }"""); + performRequestAgainstFulfillingCluster(request); + } + + private void createDataStreamIndexTemplate() throws IOException { + Request request = new Request("PUT", "_index_template/my-index-template"); + request.setJsonEntity(""" + { + "index_patterns": ["logs-*"], + "data_stream": {}, + "composed_of": ["my-template"], + "priority": 500 + }"""); + performRequestAgainstFulfillingCluster(request); + } + + private void createDataStreamDocuments() throws IOException { + Request request = new Request("POST", "logs-foo/_bulk"); + request.addParameter("refresh", ""); + request.setJsonEntity(""" + { "create" : {} } + { "@timestamp": "2099-05-06T16:21:15.000Z", "data_stream": {"namespace": "16"} } + { "create" : {} } + { "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17"} } + """); + assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); + } + + private void createDataStreamAlias() throws IOException { + Request request = new Request("PUT", "_alias"); + request.setJsonEntity(""" + { + "actions": [ + { + "add": { + "index": "logs-foo", + "alias": "alias-foo" + } + } + ] + }"""); + assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); + } + + record ExpectedCluster(String clusterAlias, String indexExpression, String status, Integer totalShards) {} +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml index b61daa068ed1a..c09f9dc620a4c 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/resources/roles.yml @@ -41,3 +41,102 @@ ccr_user_role: manage_role: cluster: [ 'manage' ] + +logs_foo_all: + cluster: [] + indices: + - names: [ 'logs-foo' ] + privileges: [ 'read' ] + remote_indices: + - names: [ 'logs-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + +logs_foo_16_only: + cluster: [] + indices: + - names: [ 'logs-foo' ] + privileges: [ 'read' ] + query: | + { + "term": { + "data_stream.namespace": "16" + } + } + remote_indices: + - names: [ 'logs-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "term": { + "data_stream.namespace": "16" + } + } + +logs_foo_after_2021: + cluster: [] + indices: + - names: [ 'logs-foo' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + remote_indices: + - names: [ 'logs-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + +logs_foo_after_2021_pattern: + cluster: [] + indices: + - names: [ 'logs-*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + remote_indices: + - names: [ 'logs-*' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + +logs_foo_after_2021_alias: + cluster: [] + indices: + - names: [ 'alias-foo' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + remote_indices: + - names: [ 'alias-foo' ] + clusters: [ '*' ] + privileges: [ 'read' ] + query: | + { + "range": { + "@timestamp": {"gte": "2021-01-01T00:00:00"} + } + } + From 43137604eec645961e21f1ab98a8233ad6c637d8 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 11 Dec 2024 10:44:46 +0100 Subject: [PATCH 2/5] Nit --- .../remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java index abfa1a6f1fb78..6ef52be015845 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java @@ -107,7 +107,7 @@ public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteCluster NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); })).around(fulfillingCluster).around(queryCluster); - public void testDataStreamWithDls() throws Exception { + public void testDataStreamsWithDls() throws Exception { configureRemoteCluster(); createDataStream(); MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); From cfe22039915a31bc3fe278275f92158f0f0630a7 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 11 Dec 2024 11:31:23 +0100 Subject: [PATCH 3/5] RCS 1 tests --- ...eClusterSecurityDataStreamEsqlRcs1IT.java} | 119 +++++++------- ...teClusterSecurityDataStreamEsqlRcs2IT.java | 155 ++++++++++++++++++ 2 files changed, 210 insertions(+), 64 deletions(-) rename x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/{RemoteClusterSecurityDataStreamEsqlIT.java => RemoteClusterSecurityDataStreamEsqlRcs1IT.java} (62%) create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java similarity index 62% rename from x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java rename to x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java index 6ef52be015845..3b44b104b9e10 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java @@ -15,7 +15,6 @@ import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.util.resource.Resource; -import org.elasticsearch.test.junit.RunnableTestRuleAdapter; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.ClassRule; @@ -24,24 +23,16 @@ import java.io.IOException; import java.util.Locale; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; -public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteClusterSecurityTestCase { - private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); - private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); - private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean(); - private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean(); - +// TODO consolidate me with RemoteClusterSecurityDataStreamEsqlRcs2IT +public class RemoteClusterSecurityDataStreamEsqlRcs1IT extends AbstractRemoteClusterSecurityTestCase { static { fulfillingCluster = ElasticsearchCluster.local() .name("fulfilling-cluster") - .nodes(3) .module("x-pack-autoscaling") .module("x-pack-esql") .module("x-pack-enrich") @@ -49,16 +40,9 @@ public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteCluster .module("x-pack-ilm") .module("ingest-common") .apply(commonClusterConfig) - .setting("remote_cluster.port", "0") .setting("xpack.ml.enabled", "false") - .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) - .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") - .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") .setting("xpack.security.authc.token.enabled", "true") - .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") - .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) - .node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get()))) - .node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get()))) + .rolesFile(Resource.fromClasspath("roles.yml")) .build(); queryCluster = ElasticsearchCluster.local() @@ -71,23 +55,7 @@ public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteCluster .module("ingest-common") .apply(commonClusterConfig) .setting("xpack.ml.enabled", "false") - .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) - .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") .setting("xpack.security.authc.token.enabled", "true") - .keystore("cluster.remote.my_remote_cluster.credentials", () -> { - if (API_KEY_MAP_REF.get() == null) { - final Map apiKeyMap = createCrossClusterAccessApiKey(""" - { - "search": [ - { - "names": ["logs-*", "alias-*"] - } - ] - }"""); - API_KEY_MAP_REF.set(apiKeyMap); - } - return (String) API_KEY_MAP_REF.get().get("encoded"); - }) .rolesFile(Resource.fromClasspath("roles.yml")) .user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false) .user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false) @@ -98,37 +66,62 @@ public class RemoteClusterSecurityDataStreamEsqlIT extends AbstractRemoteCluster } @ClassRule - // Use a RuleChain to ensure that fulfilling cluster is started before query cluster - // `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters - // We set it here, since randomization methods are not available in the static initialize context above - public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> { - SSL_ENABLED_REF.set(usually()); - NODE1_RCS_SERVER_ENABLED.set(randomBoolean()); - NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); - })).around(fulfillingCluster).around(queryCluster); + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); public void testDataStreamsWithDls() throws Exception { - configureRemoteCluster(); - createDataStream(); + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), randomBoolean()); + createDataStreamOnFulfillingCluster(); + MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); - assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), twoResults); - assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), oneResult); - assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), oneResult); assertMap( - entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult ); - assertMap(entityAsMap(runESQLCommand("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), twoResults); - assertMap(entityAsMap(runESQLCommand("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult); - assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult); - assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), oneResult); - assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)")), oneResult); - assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), oneResult); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), + oneResult + ); } - protected Response runESQLCommand(String user, String command) throws IOException { + static Response runESQLCommandAgainstQueryCluster(String user, String command) throws IOException { if (command.toLowerCase(Locale.ROOT).contains("limit") == false) { // add a (high) limit to avoid warnings on default limit command += " | limit 10000000"; @@ -176,7 +169,7 @@ static Settings randomPragmas() { return settings.build(); } - private void createDataStream() throws IOException { + static void createDataStreamOnFulfillingCluster() throws IOException { createDataStreamPolicy(); createDataStreamComponentTemplate(); createDataStreamIndexTemplate(); @@ -184,7 +177,7 @@ private void createDataStream() throws IOException { createDataStreamAlias(); } - private void createDataStreamPolicy() throws IOException { + private static void createDataStreamPolicy() throws IOException { Request request = new Request("PUT", "_ilm/policy/my-lifecycle-policy"); request.setJsonEntity(""" { @@ -210,7 +203,7 @@ private void createDataStreamPolicy() throws IOException { performRequestAgainstFulfillingCluster(request); } - private void createDataStreamComponentTemplate() throws IOException { + private static void createDataStreamComponentTemplate() throws IOException { Request request = new Request("PUT", "_component_template/my-template"); request.setJsonEntity(""" { @@ -236,7 +229,7 @@ private void createDataStreamComponentTemplate() throws IOException { performRequestAgainstFulfillingCluster(request); } - private void createDataStreamIndexTemplate() throws IOException { + private static void createDataStreamIndexTemplate() throws IOException { Request request = new Request("PUT", "_index_template/my-index-template"); request.setJsonEntity(""" { @@ -248,7 +241,7 @@ private void createDataStreamIndexTemplate() throws IOException { performRequestAgainstFulfillingCluster(request); } - private void createDataStreamDocuments() throws IOException { + private static void createDataStreamDocuments() throws IOException { Request request = new Request("POST", "logs-foo/_bulk"); request.addParameter("refresh", ""); request.setJsonEntity(""" @@ -260,7 +253,7 @@ private void createDataStreamDocuments() throws IOException { assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); } - private void createDataStreamAlias() throws IOException { + private static void createDataStreamAlias() throws IOException { Request request = new Request("PUT", "_alias"); request.setJsonEntity(""" { @@ -275,6 +268,4 @@ private void createDataStreamAlias() throws IOException { }"""); assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); } - - record ExpectedCluster(String clusterAlias, String indexExpression, String status, Integer totalShards) {} } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java new file mode 100644 index 0000000000000..59285b782de15 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.test.junit.RunnableTestRuleAdapter; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.createDataStreamOnFulfillingCluster; +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.runESQLCommandAgainstQueryCluster; + +// TODO consolidate me with RemoteClusterSecurityDataStreamEsqlRcs1IT +public class RemoteClusterSecurityDataStreamEsqlRcs2IT extends AbstractRemoteClusterSecurityTestCase { + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + private static final AtomicBoolean NODE1_RCS_SERVER_ENABLED = new AtomicBoolean(); + private static final AtomicBoolean NODE2_RCS_SERVER_ENABLED = new AtomicBoolean(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(3) + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .node(1, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE1_RCS_SERVER_ENABLED.get()))) + .node(2, spec -> spec.setting("remote_cluster_server.enabled", () -> String.valueOf(NODE2_RCS_SERVER_ENABLED.get()))) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("x-pack-ilm") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["logs-*", "alias-*"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false) + .user("logs_foo_16_only", "x-pack-test-password", "logs_foo_16_only", false) + .user("logs_foo_after_2021", "x-pack-test-password", "logs_foo_after_2021", false) + .user("logs_foo_after_2021_pattern", "x-pack-test-password", "logs_foo_after_2021_pattern", false) + .user("logs_foo_after_2021_alias", "x-pack-test-password", "logs_foo_after_2021_alias", false) + .build(); + } + + @ClassRule + // Use a RuleChain to ensure that fulfilling cluster is started before query cluster + // `SSL_ENABLED_REF` is used to control the SSL-enabled setting on the test clusters + // We set it here, since randomization methods are not available in the static initialize context above + public static TestRule clusterRule = RuleChain.outerRule(new RunnableTestRuleAdapter(() -> { + SSL_ENABLED_REF.set(usually()); + NODE1_RCS_SERVER_ENABLED.set(randomBoolean()); + NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); + })).around(fulfillingCluster).around(queryCluster); + + public void testDataStreamsWithDls() throws Exception { + configureRemoteCluster(); + createDataStreamOnFulfillingCluster(); + MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); + MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), + oneResult + ); + } +} From d76db83486f10e2c8cec1e038493b9df740eb4bd Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 11 Dec 2024 13:16:56 +0100 Subject: [PATCH 4/5] FLS coverage --- ...teClusterSecurityDataStreamEsqlRcs1IT.java | 232 ++++++++++++++---- ...teClusterSecurityDataStreamEsqlRcs2IT.java | 75 ++---- 2 files changed, 204 insertions(+), 103 deletions(-) diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java index 3b44b104b9e10..6e9fca52d693a 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.util.resource.Resource; @@ -22,6 +23,7 @@ import org.junit.rules.TestRule; import java.io.IOException; +import java.util.List; import java.util.Locale; import static org.elasticsearch.test.ListMatcher.matchesList; @@ -68,57 +70,67 @@ public class RemoteClusterSecurityDataStreamEsqlRcs1IT extends AbstractRemoteClu @ClassRule public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); - public void testDataStreamsWithDls() throws Exception { + public void testDataStreamsWithDlsAndFls() throws Exception { configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), randomBoolean()); createDataStreamOnFulfillingCluster(); + setupAdditionalUsersAndRoles(); - MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); - MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), - twoResults - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap( - runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)") - ), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - twoResults - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - oneResult - ); + doTestDataStreamsWithFlsAndDls(); + } - assertMap( - entityAsMap( - runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)") - ), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), - oneResult - ); + private void setupAdditionalUsersAndRoles() throws IOException { + createUserAndRoleOnQueryCluster("fls_user_logs_pattern", "fls_user_logs_pattern", """ + { + "indices": [ + { + "names": ["logs-*"], + "privileges": ["read"], + "field_security": { + "grant": ["@timestamp", "data_stream.namespace"] + } + } + ] + }"""); + createUserAndRoleOnFulfillingCluster("fls_user_logs_pattern", "fls_user_logs_pattern", """ + { + "indices": [ + { + "names": ["logs-*"], + "privileges": ["read"], + "field_security": { + "grant": ["@timestamp", "data_stream.namespace"] + } + } + ] + }"""); + } + + static void createUserAndRoleOnQueryCluster(String username, String roleName, String roleJson) throws IOException { + final var putRoleRequest = new Request("PUT", "/_security/role/" + roleName); + putRoleRequest.setJsonEntity(roleJson); + assertOK(adminClient().performRequest(putRoleRequest)); + + final var putUserRequest = new Request("PUT", "/_security/user/" + username); + putUserRequest.setJsonEntity(Strings.format(""" + { + "password": "%s", + "roles" : ["%s"] + }""", PASS, roleName)); + assertOK(adminClient().performRequest(putUserRequest)); + } + + static void createUserAndRoleOnFulfillingCluster(String username, String roleName, String roleJson) throws IOException { + final var putRoleRequest = new Request("PUT", "/_security/role/" + roleName); + putRoleRequest.setJsonEntity(roleJson); + assertOK(performRequestAgainstFulfillingCluster(putRoleRequest)); + + final var putUserRequest = new Request("PUT", "/_security/user/" + username); + putUserRequest.setJsonEntity(Strings.format(""" + { + "password": "%s", + "roles" : ["%s"] + }""", PASS, roleName)); + assertOK(performRequestAgainstFulfillingCluster(putUserRequest)); } static Response runESQLCommandAgainstQueryCluster(String user, String command) throws IOException { @@ -135,7 +147,9 @@ static Response runESQLCommandAgainstQueryCluster(String user, String command) t request.setJsonEntity(org.elasticsearch.common.Strings.toString(json)); request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("es-security-runas-user", user)); request.addParameter("error_trace", "true"); - return adminClient().performRequest(request); + Response response = adminClient().performRequest(request); + assertOK(response); + return response; } static void addRandomPragmas(XContentBuilder builder) throws IOException { @@ -219,7 +233,8 @@ private static void createDataStreamComponentTemplate() throws IOException { }, "data_stream": { "properties": { - "namespace": {"type": "keyword"} + "namespace": {"type": "keyword"}, + "environment": {"type": "keyword"} } } } @@ -246,9 +261,9 @@ private static void createDataStreamDocuments() throws IOException { request.addParameter("refresh", ""); request.setJsonEntity(""" { "create" : {} } - { "@timestamp": "2099-05-06T16:21:15.000Z", "data_stream": {"namespace": "16"} } + { "@timestamp": "2099-05-06T16:21:15.000Z", "data_stream": {"namespace": "16", "environment": "dev"} } { "create" : {} } - { "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17"} } + { "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17", "environment": "prod"} } """); assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); } @@ -268,4 +283,119 @@ private static void createDataStreamAlias() throws IOException { }"""); assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); } + + static void doTestDataStreamsWithFlsAndDls() throws IOException { + // DLS + MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); + MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + twoResults + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), + oneResult + ); + + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)") + ), + oneResult + ); + assertMap( + entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), + oneResult + ); + + // FLS + // logs_foo_all does not have FLS restrictions so should be able to access all fields + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | SORT data_stream.namespace | LIMIT 1") + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.environment").entry("type", "keyword"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | SORT data_stream.namespace | LIMIT 1") + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.environment").entry("type", "keyword"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster( + "fls_user_logs_pattern", + "FROM my_remote_cluster:logs-foo | SORT data_stream.namespace | LIMIT 1" + ) + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + assertMap( + entityAsMap( + runESQLCommandAgainstQueryCluster( + "fls_user_logs_pattern", + "FROM my_remote_cluster:logs-* | SORT data_stream.namespace | LIMIT 1" + ) + ), + matchesMap().extraOk() + .entry( + "columns", + List.of( + matchesMap().entry("name", "@timestamp").entry("type", "date"), + matchesMap().entry("name", "data_stream.namespace").entry("type", "keyword") + ) + ) + ); + } } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java index 59285b782de15..c5cf704177020 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs2IT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.remotecluster; -import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.junit.RunnableTestRuleAdapter; @@ -15,15 +14,14 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.io.IOException; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import static org.elasticsearch.test.ListMatcher.matchesList; -import static org.elasticsearch.test.MapMatcher.assertMap; -import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.createDataStreamOnFulfillingCluster; -import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.runESQLCommandAgainstQueryCluster; +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.createUserAndRoleOnQueryCluster; +import static org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityDataStreamEsqlRcs1IT.doTestDataStreamsWithFlsAndDls; // TODO consolidate me with RemoteClusterSecurityDataStreamEsqlRcs1IT public class RemoteClusterSecurityDataStreamEsqlRcs2IT extends AbstractRemoteClusterSecurityTestCase { @@ -101,55 +99,28 @@ public class RemoteClusterSecurityDataStreamEsqlRcs2IT extends AbstractRemoteClu NODE2_RCS_SERVER_ENABLED.set(randomBoolean()); })).around(fulfillingCluster).around(queryCluster); - public void testDataStreamsWithDls() throws Exception { + public void testDataStreamsWithDlsAndFls() throws Exception { configureRemoteCluster(); createDataStreamOnFulfillingCluster(); - MapMatcher twoResults = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(2))); - MapMatcher oneResult = matchesMap().extraOk().entry("values", matchesList().item(matchesList().item(1))); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), - twoResults - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap( - runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-foo | STATS COUNT(*)") - ), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_all", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - twoResults - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_16_only", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_pattern", "FROM my_remote_cluster:logs-* | STATS COUNT(*)")), - oneResult - ); + setupAdditionalUsersAndRoles(); - assertMap( - entityAsMap( - runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-foo | STATS COUNT(*)") - ), - oneResult - ); - assertMap( - entityAsMap(runESQLCommandAgainstQueryCluster("logs_foo_after_2021_alias", "FROM my_remote_cluster:alias-* | STATS COUNT(*)")), - oneResult - ); + doTestDataStreamsWithFlsAndDls(); + } + + private void setupAdditionalUsersAndRoles() throws IOException { + createUserAndRoleOnQueryCluster("fls_user_logs_pattern", "fls_user_logs_pattern", """ + { + "indices": [{"names": [""], "privileges": ["read"]}], + "remote_indices": [ + { + "names": ["logs-*"], + "privileges": ["read"], + "field_security": { + "grant": ["@timestamp", "data_stream.namespace"] + }, + "clusters": ["*"] + } + ] + }"""); } } From 11c6be96919eba83ae71f6863d1494e2388649ab Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 11 Dec 2024 13:27:31 +0100 Subject: [PATCH 5/5] More --- ...teClusterSecurityDataStreamEsqlRcs1IT.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java index 6e9fca52d693a..57eb583912c49 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityDataStreamEsqlRcs1IT.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.core.Strings; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -183,15 +184,15 @@ static Settings randomPragmas() { return settings.build(); } - static void createDataStreamOnFulfillingCluster() throws IOException { - createDataStreamPolicy(); - createDataStreamComponentTemplate(); - createDataStreamIndexTemplate(); - createDataStreamDocuments(); - createDataStreamAlias(); + static void createDataStreamOnFulfillingCluster() throws Exception { + createDataStreamPolicy(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamComponentTemplate(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamIndexTemplate(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamDocuments(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); + createDataStreamAlias(AbstractRemoteClusterSecurityTestCase::performRequestAgainstFulfillingCluster); } - private static void createDataStreamPolicy() throws IOException { + private static void createDataStreamPolicy(CheckedFunction requestConsumer) throws Exception { Request request = new Request("PUT", "_ilm/policy/my-lifecycle-policy"); request.setJsonEntity(""" { @@ -214,10 +215,10 @@ private static void createDataStreamPolicy() throws IOException { } }"""); - performRequestAgainstFulfillingCluster(request); + requestConsumer.apply(request); } - private static void createDataStreamComponentTemplate() throws IOException { + private static void createDataStreamComponentTemplate(CheckedFunction requestConsumer) throws Exception { Request request = new Request("PUT", "_component_template/my-template"); request.setJsonEntity(""" { @@ -241,10 +242,10 @@ private static void createDataStreamComponentTemplate() throws IOException { } } }"""); - performRequestAgainstFulfillingCluster(request); + requestConsumer.apply(request); } - private static void createDataStreamIndexTemplate() throws IOException { + private static void createDataStreamIndexTemplate(CheckedFunction requestConsumer) throws Exception { Request request = new Request("PUT", "_index_template/my-index-template"); request.setJsonEntity(""" { @@ -253,10 +254,10 @@ private static void createDataStreamIndexTemplate() throws IOException { "composed_of": ["my-template"], "priority": 500 }"""); - performRequestAgainstFulfillingCluster(request); + requestConsumer.apply(request); } - private static void createDataStreamDocuments() throws IOException { + private static void createDataStreamDocuments(CheckedFunction requestConsumer) throws Exception { Request request = new Request("POST", "logs-foo/_bulk"); request.addParameter("refresh", ""); request.setJsonEntity(""" @@ -265,10 +266,10 @@ private static void createDataStreamDocuments() throws IOException { { "create" : {} } { "@timestamp": "2001-05-06T16:21:15.000Z", "data_stream": {"namespace": "17", "environment": "prod"} } """); - assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); + assertMap(entityAsMap(requestConsumer.apply(request)), matchesMap().extraOk().entry("errors", false)); } - private static void createDataStreamAlias() throws IOException { + private static void createDataStreamAlias(CheckedFunction requestConsumer) throws Exception { Request request = new Request("PUT", "_alias"); request.setJsonEntity(""" { @@ -281,7 +282,7 @@ private static void createDataStreamAlias() throws IOException { } ] }"""); - assertMap(entityAsMap(performRequestAgainstFulfillingCluster(request)), matchesMap().extraOk().entry("errors", false)); + assertMap(entityAsMap(requestConsumer.apply(request)), matchesMap().extraOk().entry("errors", false)); } static void doTestDataStreamsWithFlsAndDls() throws IOException {