From 6ef648113d92ed5a0fde5f8de0a72e67206571c2 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Mon, 15 Sep 2025 18:16:09 +0200 Subject: [PATCH 01/22] Thorough integration tests for index API authorization Signed-off-by: Nils Bandener --- build.gradle | 5 + .../privileges/int_tests/ClusterConfig.java | 86 + ...taStreamAuthorizationReadOnlyIntTests.java | 852 +++++++++ ...aStreamAuthorizationReadWriteIntTests.java | 553 ++++++ .../IndexAuthorizationReadOnlyIntTests.java | 1546 +++++++++++++++++ .../IndexAuthorizationReadWriteIntTests.java | 1197 +++++++++++++ .../opensearch/test/framework/TestAlias.java | 142 ++ .../test/framework/TestComponentTemplate.java | 73 + .../opensearch/test/framework/TestData.java | 134 +- .../test/framework/TestDataStream.java | 117 ++ .../opensearch/test/framework/TestIndex.java | 88 +- .../TestIndexOrAliasOrDatastream.java | 58 + .../test/framework/TestIndexTemplate.java | 107 ++ .../test/framework/TestMapping.java | 52 + .../test/framework/TestSecurityConfig.java | 37 +- .../test/framework/cluster/LocalCluster.java | 62 +- .../cluster/OpenSearchClientProvider.java | 11 + .../framework/cluster/TestRestClient.java | 32 +- .../matcher/IndexApiResponseMatchers.java | 662 +++++++ .../test/framework/matcher/RestMatchers.java | 310 ++-- 20 files changed, 5889 insertions(+), 235 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java create mode 100644 src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java create mode 100644 src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/TestAlias.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/TestMapping.java create mode 100644 src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java diff --git a/build.gradle b/build.gradle index 34b797553d..28317e2761 100644 --- a/build.gradle +++ b/build.gradle @@ -571,6 +571,11 @@ allprojects { } integrationTestImplementation 'org.slf4j:slf4j-api:2.0.12' integrationTestImplementation 'com.selectivem.collections:special-collections-complete:1.4.0' + + integrationTestImplementation ('com.jayway.jsonpath:json-path:2.9.0') { + exclude(group: 'net.minidev', module: 'json-smart') + } + integrationTestImplementation 'net.minidev:json-smart:2.6.0' } } diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java new file mode 100644 index 0000000000..2d3a1627e0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.opensearch.test.framework.cluster.LocalCluster; + +/** + * This is one of the test parameter dimensions used by the *Authorization*IntTests test suites. + * The test suites run on different cluster configurations; the possible cluster configurations are defined here. + */ +public enum ClusterConfig { + LEGACY_PRIVILEGES_EVALUATION( + "legacy", + c -> c.doNotFailOnForbidden(true).nodeSettings(Map.of("plugins.security.system_indices.enabled", true)), + true, + false, + false + ), + LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION( + "legacy_system_index_perm", + c -> c.doNotFailOnForbidden(true) + .nodeSettings( + Map.of("plugins.security.system_indices.enabled", true, "plugins.security.system_indices.permission.enabled", true) + ), + true, + true, + false + ); + + final String name; + final Function clusterConfiguration; + final boolean legacyPrivilegeEvaluation; + final boolean systemIndexPrivilegeEnabled; + final boolean allowsEmptyResultSets; + + private LocalCluster cluster; + + ClusterConfig( + String name, + Function clusterConfiguration, + boolean legacyPrivilegeEvaluation, + boolean systemIndexPrivilegeEnabled, + boolean allowsEmptyResultSets + ) { + this.name = name; + this.clusterConfiguration = clusterConfiguration; + this.legacyPrivilegeEvaluation = legacyPrivilegeEvaluation; + this.systemIndexPrivilegeEnabled = systemIndexPrivilegeEnabled; + this.allowsEmptyResultSets = allowsEmptyResultSets; + } + + LocalCluster cluster(Supplier clusterBuilder) { + if (cluster == null) { + cluster = this.clusterConfiguration.apply(clusterBuilder.get()).build(); + cluster.before(); + } + return cluster; + } + + void shutdown() { + if (cluster != null) { + try { + cluster.close(); + } catch (Exception e) {} + cluster = null; + } + } + + @Override + public String toString() { + return name; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java new file mode 100644 index 0000000000..d51a5fc3b6 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -0,0 +1,852 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.TestComponentTemplate; +import org.opensearch.test.framework.TestDataStream; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.TestIndexTemplate; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read-only operations on data streams. + * It uses the following dimensions: + *
    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class DataStreamAuthorizationReadOnlyIntTests { + + static TestDataStream ds_a1 = TestDataStream.name("ds_a1").documentCount(100).rolloverAfter(10).seed(1)/*.attr("prefix", "a")*/.build(); + static TestDataStream ds_a2 = TestDataStream.name("ds_a2").documentCount(110).rolloverAfter(10).seed(2)/*.attr("prefix", "a")*/.build(); + static TestDataStream ds_a3 = TestDataStream.name("ds_a3").documentCount(120).rolloverAfter(10).seed(3)/*.attr("prefix", "a")*/.build(); + static TestDataStream ds_ax = TestDataStream.name("ds_ax").build(); // Not existing data stream + static TestDataStream ds_b1 = TestDataStream.name("ds_b1").documentCount(51).rolloverAfter(10).seed(4)/*.attr("prefix", "b")*/.build(); + static TestDataStream ds_b2 = TestDataStream.name("ds_b2").documentCount(52).rolloverAfter(10).seed(5)/*.attr("prefix", "a")*/.build(); + static TestDataStream ds_b3 = TestDataStream.name("ds_b3").documentCount(53).rolloverAfter(10).seed(6)/*.attr("prefix", "a")*/.build(); + static TestIndex index_c1 = TestIndex.name("index_c1").documentCount(5).seed(7).build(); + + static final List ALL_INDICES = List.of( + ds_a1, + ds_a2, + ds_a3, + ds_b1, + ds_b2, + ds_b3, + index_c1, + openSearchSecurityConfigIndex() + ); + + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("ds_a*")// + .roles( + // + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_a*") + )// + .indexMatcher("read", limitedTo(ds_a1, ds_a2, ds_a3, ds_ax))// + .indexMatcher("read_top_level", limitedTo(ds_a1, ds_a2, ds_a3))// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("ds_b*")// + .roles( + // + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_b*") + )// + .indexMatcher("read", limitedTo(ds_b1, ds_b2, ds_b3))// + .indexMatcher("read_top_level", limitedTo(ds_b1, ds_b2, ds_b3))// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// + .description("ds_b1")// + .roles( + // + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_b1") + )// + .indexMatcher("read", limitedTo(ds_b1))// + .indexMatcher("read_top_level", limitedTo(ds_b1))// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no privileges for existing indices")// + .roles( + // + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_does_not_exist_*") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("read_top_level", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + // + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions("*") + .on("*")// + )// + .indexMatcher("read", unlimited())// + .indexMatcher("read_top_level", unlimited())// + .indexMatcher("get_alias", unlimited()); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("read_top_level", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); + + static List USERS = List.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B1, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indexTemplates(new TestIndexTemplate("ds_test", "ds_*").dataStream().composedOf(TestComponentTemplate.DATA_STREAM_MINIMAL))// + .dataStreams(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3)// + .indices(index_c1); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void search_noPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_noPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&expand_wildcards=none"); + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_noPattern_allowNoIndicesFalse() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&allow_no_indices=false"); + + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isNotFound() : isForbidden()) + ); + } + } + + @Test + public void search_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_all_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000&expand_wildcards=none"); + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticNames_noIgnoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_search?size=1000"); + // With dnfof data streams with incomplete privileges will be replaced by their member indices + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void search_staticNames_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_search?size=1000&ignore_unavailable=true"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticIndicies_negation_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1,-.ds-ds_b1*/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-ds_b2,-ds_b3/_search?size=1000"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // does not handle the expression ds_a*,ds_b*,-ds_b2,-ds_b3 in a way that excludes the data streams. See + // search_indexPattern_minus_backingIndices for an alternative. + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The IndexResolverReplacer fails to interpret the minus patterns and falls back to interpreting the given index names + // literally + // In the logs, this then looks like this: + // | indices:data/read/search | + // -ds_b2| MISSING | + // -ds_b3| MISSING | + // ds_b* | MISSING | + // ds_a* | MISSING | + // This has the effect that granted privileges using wildcards might work, but granted privileges without wildcards won't + // work + if (user == LIMITED_USER_B1) { + // No wildcard in the index pattern + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + } + + @Test + public void search_indexPattern_minus_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-.ds-ds_b2*,-.ds-ds_b3*/_search?size=1000"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + + // dnfof has the effect that the index expression is interpreted differently and that ds_b2 and ds_b3 get included + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get( + "ds_a*,ds_b*,xxx_non_existing/_search?size=1000&ignore_unavailable=true" + ); + + // The presence of a non existing index has the effect that the other patterns are not resolved by IndexResolverReplacer + // This causes a few more 403 errors where the granted index patterns do not use wildcards + + if (user == LIMITED_USER_B1) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_indexPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get( + "ds_a*,ds_b*/_search?size=1000&expand_wildcards=none&ignore_unavailable=true" + ); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // dnfof makes the expand_wildcards=none option ineffective + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_search?size=1000"); + + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } + } + } + + @Test + public void search_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_search?size=1000"); + + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } + } + + @Test + public void search_termsAggregation_index() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("_search", """ + { + "size": 0, + "aggs": { + "indices": { + "terms": { + "field": "_index", + "size": 1000 + } + } + } + }"""); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("aggregations.indices.buckets[*].key") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isOk()) + ); + } else { + // Users without full privileges will not see hidden indices here; thus on a cluster with only data streams, the result is + // often just empty + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("aggregations.indices.buckets[*].key")); + } + } + } + + @Test + public void msearch_staticIndices() throws Exception { + String msearchBody = """ + {"index": "ds_b1"} + {"size": 10, "query": {"bool":{"must":{"match_all":{}}}}} + {"index": "ds_b2"} + {"size": 10, "query": {"bool":{"must":{"match_all":{}}}}} + """; + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("_msearch", msearchBody); + assertThat( + httpResponse, + containsExactly(ds_b1, ds_b2).at("responses[*].hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isOk()) + ); + } + } + + @Test + public void index_stats_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_stats"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("indices.keys()") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void index_stats_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_b*/_stats"); + assertThat( + httpResponse, + containsExactly(ds_b1, ds_b2, ds_b3).at("indices.keys()") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void getDataStream_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].name") + .butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStream_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/*"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].name") + .butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStream_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a*"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStream_pattern_negation() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_*,-ds_b*"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStream_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a1,ds_a2"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStreamStats_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/_stats"); + // The legacy mode does not support dnfof for indices:monitor/data_stream/stats + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].data_stream") + .butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStreamStats_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/*/_stats"); + // The legacy mode does not support dnfof for indices:monitor/data_stream/stats + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].data_stream") + .butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStreamStats_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a*/_stats"); + // The legacy mode does not support dnfof for indices:monitor/data_stream/stats + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void getDataStreamStats_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a1,ds_a2/_stats"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.indexMatcher("read")) + ); + } + } + + @Test + public void resolve_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("$.*[*].name") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void resolve_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/ds_a*,ds_b*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.*[*].name") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_staticIndices_noIgnoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("indices").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + + } + } + + @Test + public void field_caps_staticIndices_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_field_caps?fields=*&ignore_unavailable=true"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_field_caps?fields=*"); + + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse.getStatusCode(), is(403)); + } + } + } + + @Test + public void field_caps_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_field_caps?fields=*"); + + assertThat(httpResponse, containsExactly().at("indices").whenEmpty(isOk())); + } + } + + @Test + public void field_caps_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-ds_b2,-ds_b3/_field_caps?fields=*"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // OpenSearch does not handle the expression ds_a*,ds_b*,-ds_b2,-ds_b3 in a way that excludes the data streams. See + // field_caps_indexPattern_minus_backingIndices for an alternative. + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + if (user == LIMITED_USER_B1) { + // No wildcard in the index pattern + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + } + + @Test + public void field_caps_indexPattern_minus_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-.ds-ds_b2*,-.ds-ds_b3*/_field_caps?fields=*"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // dnfof has the effect that the index expression is interpreted differently and that ds_b2 and ds_b3 get included + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void field_caps_staticIndices_negation_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1,-.ds-ds_b1*/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public DataStreamAuthorizationReadOnlyIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(DataStreamAuthorizationReadOnlyIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java new file mode 100644 index 0000000000..7d28b8c81c --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java @@ -0,0 +1,553 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import javax.annotation.concurrent.NotThreadSafe; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.TestComponentTemplate; +import org.opensearch.test.framework.TestDataStream; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.TestIndexTemplate; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; +import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; +import static org.junit.Assert.assertEquals; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read/write operations on data streams. + * It uses the following dimensions: + *
    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@NotThreadSafe +public class DataStreamAuthorizationReadWriteIntTests { + static TestDataStream ds_ar1 = TestDataStream.name("ds_ar1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_ar2 = TestDataStream.name("ds_ar2").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_aw1 = TestDataStream.name("ds_aw1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_aw2 = TestDataStream.name("ds_aw2").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_br1 = TestDataStream.name("ds_br1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_br2 = TestDataStream.name("ds_br2").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_bw1 = TestDataStream.name("ds_bw1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_bw2 = TestDataStream.name("ds_bw2").documentCount(22).rolloverAfter(10).build(); + static TestIndex index_cr1 = TestIndex.name("index_cr1").documentCount(10).build(); + static TestIndex index_cw1 = TestIndex.name("index_cw1").documentCount(10).build(); + static TestDataStream ds_hidden = TestDataStream.name("ds_hidden").documentCount(10).rolloverAfter(3).seed(8).build(); + + static TestDataStream ds_bwx1 = TestDataStream.name("ds_bwx1").documentCount(0).build(); // not initially created + static TestDataStream ds_bwx2 = TestDataStream.name("ds_bwx2").documentCount(0).build(); // not initially created + + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("ds_a*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_a*")// + .indexPermissions("write") + .on("ds_aw*") + )// + .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .indexMatcher("write", limitedTo(ds_aw1, ds_aw2))// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("ds_b*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*") + )// + .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B_READ_ONLY_A = new TestSecurityConfig.User("limited_user_B_read_only_A")// + .description("ds_b*; read only on ds_a*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_a*", "ds_b*")// + .indexPermissions("write") + .on("ds_bw*") + )// + .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + /** + * This is an artificial user - in the sense that in real life it would likely not exist this way. + * It has privileges to write on ds_b*, but privileges for indices:admin/mapping/auto_put on all data streams. + * The reason is that some indexing operations are two phase - first auto put, then indexing. To be able to test both + * phases, we need which user which always allows the first phase to pass. + */ + static TestSecurityConfig.User LIMITED_USER_B_AUTO_PUT_ON_ALL = new TestSecurityConfig.User("limited_user_B_auto_put_on_all")// + .description("ds_b* with full auto put")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*")// + .indexPermissions("indices:admin/mapping/auto_put") + .on("*") + )// + .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B_CREATE_DS = new TestSecurityConfig.User("limited_user_B_create_ds")// + .description("ds_b* with create ds privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*")// + .indexPermissions("indices:admin/data_stream/create") + .on("ds_bw*") + )// + .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("create_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B_MANAGE_DS = new TestSecurityConfig.User("limited_user_B_manage_ds")// + .description("ds_b* with manage privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*")// + .indexPermissions("manage") + .on("ds_bw*") + )// + .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("create_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("manage_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + + static TestSecurityConfig.User LIMITED_USER_AB_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_AB_manage_index")// + .description("ds_a*, ds_b* with manage index privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("ds_a*", "ds_b*")// + .indexPermissions("write") + .on("ds_aw*", "ds_bw*")// + .indexPermissions("manage") + .on("ds_aw*", "ds_bw*") + )// + .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("write", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("create_data_stream", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .indexMatcher("manage_data_stream", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + + static TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// + .description("index_c*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh") + .on("index_c*")// + .indexPermissions("write") + .on("index_cw*") + )// + .indexMatcher("read", limitedTo(index_cr1, index_cw1))// + .indexMatcher("write", limitedTo(index_cw1))// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// + .description("read/only on *")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("*") + )// + .indexMatcher("read", unlimited())// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// + .description("read/only on ds_a*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("ds_a*") + )// + .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// + .description("no privileges for existing indices")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("crud", "indices_monitor") + .on("ds_does_not_exist_*") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES = new TestSecurityConfig.User( + "limited_user_permissions_on_backing_indices" + )// + .description("ds_a* on backing indices")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on(".ds-ds_a*")// + .indexPermissions("write") + .on(".ds-ds_aw*") + )// + .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .indexMatcher("write", limitedTo(ds_aw1, ds_aw2))// + .indexMatcher("create_data_stream", limitedToNone())// + .indexMatcher("manage_data_stream", limitedToNone()); + + static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("*") + .on("*") + )// + .indexMatcher("read", unlimited())// + .indexMatcher("write", unlimited())// + .indexMatcher("create_data_stream", unlimited())// + .indexMatcher("manage_data_stream", unlimited()); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("write", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("create_data_stream", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("manage_data_stream", unlimitedIncludingOpenSearchSecurityIndex()); + + static List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B_READ_ONLY_A, + LIMITED_USER_B_AUTO_PUT_ON_ALL, + LIMITED_USER_B_CREATE_DS, + LIMITED_USER_B_MANAGE_DS, + LIMITED_USER_AB_MANAGE_INDEX, + LIMITED_USER_C, + LIMITED_READ_ONLY_ALL, + LIMITED_READ_ONLY_A, + LIMITED_USER_OTHER_PRIVILEGES, + LIMITED_USER_NONE, + LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indexTemplates(new TestIndexTemplate("ds_test", "ds_*").dataStream().composedOf(TestComponentTemplate.DATA_STREAM_MINIMAL))// + .indices(index_cr1, index_cw1)// + .dataStreams(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_hidden)// + .plugin(IndexAuthorizationReadOnlyIntTests.SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void createDocument() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("ds_bw1/_doc/", json("a", 1, "@timestamp", Instant.now().toString())); + assertThat(httpResponse, containsExactly(ds_bw1).at("_index").reducedBy(user.indexMatcher("write")).whenEmpty(isForbidden())); + } + } + + @Test + public void deleteByQuery_indexPattern() throws Exception { + String testName = "deleteByQuery_indexPattern"; + + try (TestRestClient restClient = cluster.getRestClient(user)) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + // Init test data + HttpResponse httpResponse = adminRestClient.put( + "ds_bw1/_create/put_delete_delete_by_query_b1?refresh=true", + json("test", testName, "delete_by_query_test_delete", "yes", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "ds_bw1/_create/put_delete_delete_by_query_b2?refresh=true", + json("test", testName, "delete_by_query_test_delete", "no", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "ds_aw1/_create/put_delete_delete_by_query_a1?refresh=true", + json("test", testName, "delete_by_query_test_delete", "yes", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "ds_aw1/_create/put_delete_delete_by_query_a2?refresh=true", + json("test", testName, "delete_by_query_test_delete", "no", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + } + + HttpResponse httpResponse = restClient.postJson("ds_aw*,ds_bw*/_delete_by_query?refresh=true&wait_for_completion=true", """ + { + "query": { + "term": { + "delete_by_query_test_delete": "yes" + } + } + } + """); + + if (clusterConfig.legacyPrivilegeEvaluation) { + // dnfof is not applicable to indices:data/write/delete/byquery, so we need privileges for all indices + if (user.indexMatcher("write").coversAll(ds_aw1, ds_aw2, ds_bw1, ds_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else { + if (user != LIMITED_USER_NONE && user != LIMITED_READ_ONLY_ALL && user != LIMITED_READ_ONLY_A) { + assertThat(httpResponse, isOk()); + int expectedDeleteCount = containsExactly(ds_aw1, ds_bw1).at("_index").reducedBy(user.indexMatcher("write")).size(); + assertEquals(httpResponse.getBody(), expectedDeleteCount, httpResponse.bodyAsMap().get("deleted")); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + deleteTestDocs(testName, "ds_aw*,ds_bw*"); + } + } + + @Test + public void putDocument_bulk() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("_bulk?refresh=true", """ + { "create": { "_index": "ds_aw1", "_id": "d1" } } + { "a": 1, "test": "putDocument_bulk", "@timestamp": "2025-09-15T12:00:00Z" } + { "create": { "_index": "ds_bw1", "_id": "d1" } } + { "b": 1, "test": "putDocument_bulk", "@timestamp": "2025-09-15T12:00:01Z" } + """); + + if (user == LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES) { + // IndexResolverReplacer won't resolve data stream names to member index names, because it does not + // specify the includeDataStream option and thus just stumbles over an IndexNotFoundException + // Thus, in contrast to aliases, privileges on backing index names won't work + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("items[*].create[?(@.result == 'created')]._index")); + } else if (user != LIMITED_USER_NONE) { + assertThat( + httpResponse, + containsExactly(ds_aw1, ds_bw1).at("items[*].create[?(@.result == 'created')]._index") + .reducedBy(user.indexMatcher("write")) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deleteTestDocs("putDocument_bulk", "ds_aw*,ds_bw*"); + } + } + + @Test + public void createDataStream() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("_data_stream/ds_bwx1"); + + if (containsExactly(ds_bwx1).reducedBy(user.indexMatcher("create_data_stream")).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + delete(ds_bwx1); + } + } + + @Test + public void putDataStream() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("ds_bwx1/", "{}"); + + if (user == UNLIMITED_USER + || user == SUPER_UNLIMITED_USER + || user == LIMITED_USER_B_MANAGE_DS + || user == LIMITED_USER_AB_MANAGE_INDEX) { + // This will fail because we try to create an index under a name of a data stream + assertThat(httpResponse, isBadRequest()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(ds_bwx1); + } + } + + @Test + public void deleteDataStream() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(ds_bwx1); + + HttpResponse httpResponse = restClient.delete("_data_stream/ds_bwx1"); + + if (user.indexMatcher("manage_data_stream").isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + delete(ds_bwx1); + } + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public DataStreamAuthorizationReadWriteIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(DataStreamAuthorizationReadWriteIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + private void createInitialTestObjects(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.createInitialTestObjects(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void delete(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.delete(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void deleteTestDocs(String testName, String indices) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + adminRestClient.postJson(indices + "/_delete_by_query?refresh=true&wait_for_completion=true", """ + { + "query": { + "term": { + "test.keyword": "%s" + } + } + } + """.formatted(testName)); + } catch (Exception e) { + throw new RuntimeException("Error while cleaning up test docs", e); + } + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java new file mode 100644 index 0000000000..866926d741 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -0,0 +1,1546 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.common.settings.Settings; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SystemIndexPlugin; +import org.opensearch.test.framework.TestAlias; +import org.opensearch.test.framework.TestData; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.IndexMatcher; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; +import static org.junit.Assert.assertTrue; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read-only operations on indices and aliases. + * It uses the following dimensions: + *
    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class IndexAuthorizationReadOnlyIntTests { + + static final TestIndex index_a1 = TestIndex.name("index_a1").documentCount(100).seed(1).build(); + static final TestIndex index_a2 = TestIndex.name("index_a2").documentCount(110).seed(2).build(); + static final TestIndex index_a3 = TestIndex.name("index_a3").documentCount(120).seed(3).build(); + static final TestIndex index_ax = TestIndex.name("index_ax").build(); // Not existing index + static final TestIndex index_b1 = TestIndex.name("index_b1").documentCount(51).seed(4).build(); + static final TestIndex index_b2 = TestIndex.name("index_b2").documentCount(52).seed(5).build(); + static final TestIndex index_b3 = TestIndex.name("index_b3").documentCount(53).seed(6).build(); + static final TestIndex index_c1 = TestIndex.name("index_c1").documentCount(5).seed(7).build(); + static final TestIndex index_hidden = TestIndex.name("index_hidden").hidden().documentCount(1).seed(8).build(); + static final TestIndex index_hidden_dot = TestIndex.name(".index_hidden_dot").hidden().documentCount(1).seed(8).build(); + static final TestIndex system_index_plugin = TestIndex.name(".system_index_plugin").hidden().documentCount(1).seed(8).build(); + + static final TestAlias alias_ab1 = new TestAlias("alias_ab1", index_a1, index_a2, index_a3, index_b1); + static final TestAlias alias_c1 = new TestAlias("alias_c1", index_c1); + + static final List ALL_INDICES_EXCEPT_SYSTEM_INDICES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot + ); + + static final List ALL_INDICES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin, + openSearchSecurityConfigIndex() + ); + + static final List ALL_INDICES_AND_ALIASES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot, + system_index_plugin, + openSearchSecurityConfigIndex() + ); + + static final TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("index_a*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_a*") + )// + .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax))// + .indexMatcher("search", limitedTo(index_a1, index_a2, index_a3, index_ax))// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("index_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_b*") + )// + .indexMatcher("read", limitedTo(index_b1, index_b2, index_b3))// + .indexMatcher("search", limitedTo(index_b1, index_b2, index_b3))// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// + .description("index_b1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_b1") + )// + .indexMatcher("read", limitedTo(index_b1))// + .indexMatcher("search", limitedTo(index_b1))// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// + .description("index_c*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_c*") + )// + .indexMatcher("read", limitedTo(index_c1, alias_c1))// + .indexMatcher("search", limitedTo(index_c1, alias_c1))// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_ALIAS_AB1 = new TestSecurityConfig.User("limited_user_alias_AB1")// + .description("alias_ab1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") + .on("alias_ab1*") + )// + .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// + .indexMatcher("search", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// + .indexMatcher("get_alias", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1)); + + static final TestSecurityConfig.User LIMITED_USER_ALIAS_C1 = new TestSecurityConfig.User("limited_user_alias_C1")// + .description("alias_c1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") + .on("alias_c1") + )// + .indexMatcher("read", limitedTo(index_c1, alias_c1))// + .indexMatcher("search", limitedTo(index_c1, alias_c1))// + .indexMatcher("get_alias", limitedTo(index_c1, alias_c1)); + + static final TestSecurityConfig.User LIMITED_USER_A_HIDDEN = new TestSecurityConfig.User("limited_user_A_hidden")// + .description("index_a*, index_hidden*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_a*", "index_hidden*", ".index_hidden*") + )// + .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// + .indexMatcher("search", limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_C_WITH_SYSTEM_INDICES = new TestSecurityConfig.User( + "limited_user_C_with_system_indices" + )// + .description("index_c*, .system_index_plugin")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_c*")// + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "system:admin/system_index") + .on(".system_index_plugin") + )// + .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin))// + .indexMatcher("search", limitedTo(index_c1, alias_c1, system_index_plugin))// + .indexMatcher("get_alias", limitedToNone()); + + /** + * This user has no privileges for indices that are used in this test. But they have privileges for other indices. + * This allows them to use actions like _search and receive empty result sets. + *

+ * Compare with LIMITED_USER_NONE, which has no search privileges and will only receive 403 errors. + */ + static final TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_index_privileges")// + .description("no privileges for tested indices")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("crud", "indices_monitor", "indices:admin/analyze") + .on("index_does_not_exist_*") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("search", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("search", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("*") + .on("*")// + + )// + .indexMatcher( + "read", + limitedTo( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot, + index_ax + ) + )// + .indexMatcher( + "search", + limitedTo( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot, + index_ax + ) + )// + .indexMatcher( + "get_alias", + limitedTo( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot, + index_ax + ) + ); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static final TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("search", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); + + static final List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B1, + LIMITED_USER_C, + LIMITED_USER_ALIAS_AB1, + LIMITED_USER_ALIAS_C1, + LIMITED_USER_A_HIDDEN, + LIMITED_USER_C_WITH_SYSTEM_INDICES, + LIMITED_USER_OTHER_PRIVILEGES, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indices( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + )// + .aliases(alias_ab1, alias_c1)// + .plugin(SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void search_noPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_noPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("/_search?size=1000&expand_wildcards=none"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_noPattern_allowNoIndicesFalse() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("/_search?size=1000&allow_no_indices=false"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void search_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_all_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000&expand_wildcards=none"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_all_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000&expand_wildcards=all"); + + assertThat( + httpResponse, + containsExactly( + clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER + ? ALL_INDICES + : ALL_INDICES_EXCEPT_SYSTEM_INDICES + ).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_wildcard_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000&expand_wildcards=none"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_wildcard_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000&expand_wildcards=all"); + + assertThat( + httpResponse, + containsExactly( + clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER + ? ALL_INDICES + : ALL_INDICES_EXCEPT_SYSTEM_INDICES + ).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("search")).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void search_staticIndices_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a1,index_b1/_search?size=1000&ignore_unavailable=true"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticIndices_nonExisting() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_search?size=1000"); + + if (containsExactly(index_ax).reducedBy(user.indexMatcher("search")).isEmpty()) { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } else { + assertThat(httpResponse, isNotFound()); + } + } + } + + @Test + public void search_staticIndices_hidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(index_hidden).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("search")) + ); + } + } + + @Test + public void search_staticIndices_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get(".system_index_plugin/_search?size=1000"); + if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("search")) + ); + } else { + // legacy privilege evaluation without system index privilege enabled + if (user == UNLIMITED_USER || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, + // but withholds documents on the DLS level + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } + } + } + } + + @Test + public void search_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*,-index_b2,-index_b3/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get( + "index_a*,index_b*,xxx_non_existing/_search?size=1000&ignore_unavailable=true" + ); + + // The presence of a non existing index has the effect that the other patterns are not resolved by IndexResolverReplacer + // This causes a few more 403 errors where the granted index patterns do not use wildcards + + if (user == LIMITED_USER_B1 || user == LIMITED_USER_ALIAS_AB1) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_indexPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*/_search?size=1000&expand_wildcards=none"); + // We have to specify the users here explicitly because here we need to check privileges for the + // non-existing (and invalidly named) indices "index_a*" and "index_b*". + // However: Again, dnfof gets the indices options wrong and ignores the expand_wildcards=none flag when getting active + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + // Only these users "get through". Because the indices does not exist, they get a 404 + assertThat(httpResponse, isNotFound()); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + + } + } + + @Test + public void search_indexPatternAndStatic_negation() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + // If there is a wildcard, negation will also affect indices specified without a wildcard + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b1,index_b2,-index_b2/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1/_search?size=1000"); + if (clusterConfig.legacyPrivilegeEvaluation) { + // The legacy privilege evaluation with dnfof enabled can replace aliases by a sub-set of its member indices + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void search_alias_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1*/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_alias_pattern_negation() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_*,-alias_ab1/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_aliasAndIndex_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1,index_b1/_search?size=1000&ignore_unavailable=true"); + if (clusterConfig.legacyPrivilegeEvaluation) { + // The legacy privilege evaluation with dnfof enabled can replace aliases by a sub-set of its member indices + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(isForbidden()) + ); + + } + } + } + + @Test + public void search_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_search?size=1000"); + // TODO adapt name to match privs for some others + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } + } + } + + @Test + public void search_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_search?size=1000"); + + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index").whenEmpty(isOk())); + } + } + + @Test + public void search_termsAggregation_index() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson( + "/_search", + "{\"size\":0,\"aggs\":{\"indices\":{\"terms\":{\"field\":\"_index\",\"size\":1000}}}}" + ); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at( + "aggregations.indices.buckets[*].key" + ).reducedBy(user.indexMatcher("search")).whenEmpty(isOk()) + ); + } + } + + @Test + public void search_pit() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("index_a*,index_b*/_search/point_in_time?keep_alive=1m"); + + IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher = containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3 + ); + + if (indexMatcher.reducedBy(user.indexMatcher("search")).isEmpty()) { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } else { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + assertThat(httpResponse, isOk()); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("search"))); + } + } + } + + @Test + public void search_pit_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("_all/_search/point_in_time?keep_alive=1m"); + + IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher; + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + indexMatcher = containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1); + } else { + indexMatcher = containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot + ); + } + + if (indexMatcher.reducedBy(user.indexMatcher("search")).isEmpty()) { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } else { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + if (clusterConfig.systemIndexPrivilegeEnabled && user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // The current request mixes access to a normal index and a system index. + // The current system index permission implementation has the issue that it also + // expects the system index permission for the normal issue then. + // As this is not present, the request https://github.com/opensearch-project/security/issues/5508 + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("search"))); + } + } + } + } + + @Test + public void search_pit_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("index_a1/_search/point_in_time?keep_alive=1m"); + + IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher = containsExactly(index_a1); + + if (indexMatcher.reducedBy(user.indexMatcher("search")).isEmpty()) { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } else { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("search"))); + } + } + } + + @Test + public void search_pit_wrongIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("index_a*/_search/point_in_time?keep_alive=1m"); + + if (user.indexMatcher("search").coversAll(index_a1, index_a2, index_a3)) { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("index_b*/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + assertThat(httpResponse, isBadRequest("/error/root_cause/0/reason", "[indices] cannot be used with point in time")); + + } else { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } + } + } + + @Test + public void msearch_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_msearch", """ + {"index":"index_b1"} + {"size":10, "query":{"bool":{"must":{"match_all":{}}}}} + {"index":"index_b2"} + {"size":10, "query":{"bool":{"must":{"match_all":{}}}}} + """); + assertThat( + httpResponse, + containsExactly(index_b1, index_b2).at("responses[*].hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isOk()) + ); + } + } + + @Test + public void mget() throws Exception { + TestData.TestDocument testDocumentA1 = index_a1.anyDocument(); + TestData.TestDocument testDocumentB1 = index_b1.anyDocument(); + TestData.TestDocument testDocumentB2 = index_b2.anyDocument(); + + String mget = String.format(""" + { + "docs": [ + { "_index": "index_a1", "_id": "%s" }, + { "_index": "index_b1", "_id": "%s" }, + { "_index": "index_b2", "_id": "%s" } + ] + } + """, testDocumentA1.id(), testDocumentB1.id(), testDocumentB2.id()); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_mget", mget); + assertThat( + httpResponse, + containsExactly(index_a1, index_b1, index_b2).at("docs[?(@.found == true)]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isOk()) + ); + } + } + + @Test + public void mget_alias() throws Exception { + TestData.TestDocument testDocumentC1a = index_c1.anyDocument(); + TestData.TestDocument testDocumentC1b = index_c1.anyDocument(); + + String mget = String.format(""" + { + "docs": [ + { "_index": "alias_c1", "_id": "%s" }, + { "_index": "alias_c1", "_id": "%s" } + ] + } + """, testDocumentC1a.id(), testDocumentC1b.id()); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_mget", mget); + assertThat( + httpResponse, + containsExactly(index_c1).at("docs[?(@.found == true)]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isOk()) + ); + } + } + + @Test + public void get() throws Exception { + TestData.TestDocument testDocumentB1 = index_b1.anyDocument(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_b1/_doc/" + testDocumentB1.id()); + assertThat(httpResponse, containsExactly(index_b1).at("_index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + } + } + + @Test + public void get_alias() throws Exception { + TestData.TestDocument testDocumentC1 = index_c1.anyDocument(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_c1/_doc/" + testDocumentC1.id()); + assertThat(httpResponse, containsExactly(index_c1).at("_index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + } + } + + @Test + public void get_systemIndex() throws Exception { + TestData.TestDocument testDocument = system_index_plugin.anyDocument(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get(".system_index_plugin/_doc/" + testDocument.id()); + + if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(system_index_plugin).at("_index")); + } else if (user == LIMITED_USER_C_WITH_SYSTEM_INDICES || user == UNLIMITED_USER) { + // If the user has a role that grants access to the index, they can + // successfully access the index (i.e., they won't get a 403), but + // the index will appear empty (i.e., they will get a 404) + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else if ((clusterConfig.systemIndexPrivilegeEnabled && user == LIMITED_USER_C_WITH_SYSTEM_INDICES) + || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(system_index_plugin).at("_index")); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void cat_indices_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/indices?format=json"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("$[*].index") + .reducedBy(user.indexMatcher("read")) + ); + + } else { + // Also here, dnfof might introduce hidden indices even though they were not requested + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden_dot, + index_hidden, + system_index_plugin + ).at("$[*].index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void cat_indices_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/indices/index_a*?format=json"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3).at("$[*].index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void cat_indices_all_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/indices?format=json&expand_wildcards=all"); + if (user == UNLIMITED_USER) { + assertThat(httpResponse, containsExactly(ALL_INDICES).at("$[*].index")); + } else { + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$[*].index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void cat_aliases_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases?format=json"); + + if (!user.indexMatcher("get_alias").isEmpty()) { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1).at("$[*].alias").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void cat_aliases_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases/alias_a*?format=json"); + + if (!user.indexMatcher("get_alias").isEmpty()) { + assertThat( + httpResponse, + containsExactly(alias_ab1).at("$[*].alias").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void index_stats_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("/_stats"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices.keys()") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // Also here, dnfof can introduce hidden indices even though they were not requested + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("indices.keys()") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void index_stats_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_b*/_stats"); + + assertThat( + httpResponse, + containsExactly(index_b1, index_b2, index_b3).at("indices.keys()") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void getAlias_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_alias"); + System.out.println(httpResponse.getBody()); + if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { + // The legacy privilege evaluation also allows regular users access to metadata of the security index + // This is not a security issue, as the metadata are not really security relevant + assertThat(httpResponse, containsExactly(ALL_INDICES).at("$.keys()")); + } else { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1).at("$.*.aliases.keys()") + .reducedBy(user.indexMatcher("get_alias")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$.keys()") + .reducedBy(user.indexMatcher("get_alias")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void getAlias_staticAlias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_alias/alias_c1"); + if (user == LIMITED_USER_ALIAS_AB1) { + if (clusterConfig.legacyPrivilegeEvaluation) { + // RestGetAliasesAction does some further post processing on the results, thus we get 404 errors in case a non wildcard + // alias was removed + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/aliases/get]")); + } + } else { + assertThat( + httpResponse, + containsExactly(alias_c1).at("$.*.aliases.keys()").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isForbidden()) + ); + assertThat( + httpResponse, + containsExactly(index_c1).at("$.keys()").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void getAlias_aliasPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_alias/alias_ab*"); + + if (user == LIMITED_USER_ALIAS_AB1 || user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(alias_ab1).at("$.*.aliases.keys()").reducedBy(user.indexMatcher("get_alias"))); + assertThat(httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("$.keys()")); + } else if (user == LIMITED_USER_ALIAS_C1) { + // This is also a kind of anomaly in the legacy privilege evaluation: Even though we do not have permissions + // we get a 200 response with an empty result + assertThat(httpResponse, isOk()); + assertTrue(httpResponse.getBody(), httpResponse.bodyAsMap().isEmpty()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/aliases/get]")); + } + + } + } + + @Test + public void analyze_noIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("_analyze", "{\"text\": \"sample text\"}"); + + // _analyze without index is different from most other operations: + // Usually, the absence of an index means "all indices". For analyze, however, it means: "no index". + // However, the IndexResolverReplacer does not get this right; it assumes that all indices are requested. + // Thus, we get only through to this operation with full privileges for all indices + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/analyze]")); + } + } + } + + @Test + public void analyze_staticIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("index_a1/_analyze", "{\"text\": \"sample text\"}"); + IndexMatcher matcher = containsExactly(index_a1).reducedBy(user.indexMatcher("read")); + + if (matcher.isEmpty()) { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/analyze]")); + } else { + assertThat(httpResponse, isOk()); + } + } + } + + @Test + public void resolve_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, alias_ab1, alias_c1).at( + "$.*[*].name" + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("$.*[*].name") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + + } + } + + @Test + public void resolve_wildcard_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*?expand_wildcards=all"); + + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + // The legacy privilege evaluation also allows regular users access to metadata of the security index + // This is not a security issue, as the metadata are not really security relevant + assertThat(httpResponse, containsExactly(ALL_INDICES_AND_ALIASES).at("$.*[*].name")); + } else { + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$.*[*].name").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void resolve_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/index_a*,index_b*"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("$.*[*].name") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + } + + @Test + public void field_caps_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + TestRestClient.HttpResponse httpResponse = restClient.get("/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } else { + TestRestClient.HttpResponse httpResponse = restClient.get("/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + + } + } + + @Test + public void field_caps_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_b*/_field_caps?fields=*"); + + assertThat( + httpResponse, + containsExactly(index_b1, index_b2, index_b3).at("indices").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void field_caps_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_field_caps?fields=*"); + assertThat(httpResponse, containsExactly(index_a1).at("indices").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + } + } + + @Test + public void field_caps_staticIndices_hidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_field_caps?fields=*"); + assertThat(httpResponse, containsExactly(index_hidden).at("indices").butForbiddenIfIncomplete(user.indexMatcher("read"))); + } + } + + @Test + public void field_caps_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1/_field_caps?fields=*"); + if (clusterConfig.legacyPrivilegeEvaluation) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void field_caps_aliasPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab*/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + } + + @Test + public void field_caps_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_field_caps?fields=*"); + + if (containsExactly(index_ax).reducedBy(user.indexMatcher("read")).isEmpty()) { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/field_caps]")); + } else { + assertThat(httpResponse, isNotFound()); + } + } + } + + @Test + public void field_caps_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_field_caps?fields=*"); + + // As this resolves to an empty set of indices, we are always allowed + assertThat(httpResponse, containsExactly().at("indices").whenEmpty(isOk())); + } + } + + @Test + public void field_caps_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*,-index_b2,-index_b3/_field_caps?fields=*"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public IndexAuthorizationReadOnlyIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(IndexAuthorizationReadOnlyIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return List.of( + new SystemIndexDescriptor(".system_index_plugin", "for testing system index exclusion"), + new SystemIndexDescriptor(".system_index_plugin_not_existing", "for testing system index exclusion") + ); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java new file mode 100644 index 0000000000..a1e779b933 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -0,0 +1,1197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import javax.annotation.concurrent.NotThreadSafe; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.indices.open.OpenIndexRequest; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.framework.TestAlias; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; +import org.opensearch.transport.client.Client; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.*; +import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read/write operations on indices and aliases. + * It uses the following dimensions: + *

    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@NotThreadSafe +public class IndexAuthorizationReadWriteIntTests { + + static final TestIndex index_ar1 = TestIndex.name("index_ar1").documentCount(10).build(); + static final TestIndex index_ar2 = TestIndex.name("index_ar2").documentCount(10).build(); + static final TestIndex index_aw1 = TestIndex.name("index_aw1").documentCount(10).build(); + static final TestIndex index_aw2 = TestIndex.name("index_aw2").documentCount(10).build(); + static final TestIndex index_br1 = TestIndex.name("index_br1").documentCount(10).build(); + static final TestIndex index_br2 = TestIndex.name("index_br2").documentCount(10).build(); + static final TestIndex index_bw1 = TestIndex.name("index_bw1").documentCount(10).build(); + static final TestIndex index_bw2 = TestIndex.name("index_bw2").documentCount(10).build(); + static final TestIndex index_cr1 = TestIndex.name("index_cr1").documentCount(10).build(); + static final TestIndex index_cw1 = TestIndex.name("index_cw1").documentCount(10).build(); + static final TestIndex index_hidden = TestIndex.name("index_hidden").hidden().documentCount(1).seed(8).build(); + static final TestIndex system_index_plugin = TestIndex.name(".system_index_plugin").hidden().documentCount(1).seed(8).build(); + static final TestIndex system_index_plugin_not_existing = TestIndex.name(".system_index_plugin_not_existing") + .hidden() + .documentCount(0) + .build(); // not initially created + + static final TestAlias alias_ab1r = new TestAlias("alias_ab1r", index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1); + static final TestAlias alias_ab1w = new TestAlias("alias_ab1w", index_aw1, index_aw2, index_bw1).writeIndex(index_aw1); + static final TestAlias alias_ab1w_nowriteindex = new TestAlias("alias_ab1w_nowriteindex", index_aw1, index_aw2, index_bw1); + + static final TestAlias alias_c1 = new TestAlias("alias_c1", index_cr1, index_cw1); + + static final TestIndex index_bwx1 = TestIndex.name("index_bwx1").documentCount(0).build(); // not initially created + static final TestIndex index_bwx2 = TestIndex.name("index_bwx2").documentCount(0).build(); // not initially created + + static final TestAlias alias_bwx = new TestAlias("alias_bwx"); // not initially created + + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("index_a*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_a*")// + .indexPermissions("write") + .on("index_aw*") + )// + .indexMatcher("read", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// + .indexMatcher("write", limitedTo(index_aw1, index_aw2))// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("index_b*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*") + )// + .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B_CREATE_INDEX = new TestSecurityConfig.User("limited_user_B_create_index")// + .description("index_b* with create index privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*")// + .indexPermissions("create_index") + .on("index_bw*") + )// + .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_B_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_B_manage_index")// + .description("index_b* with manage privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*")// + .indexPermissions("manage") + .on("index_bw*") + )// + .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("get_alias", limitedTo()); + + static TestSecurityConfig.User LIMITED_USER_B_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User("limited_user_B_manage_index_alias")// + .description("index_b*, alias_bwx* with manage privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*")// + .indexPermissions("manage") + .on("index_bw*")// + .indexPermissions("manage_aliases") + .on("alias_bwx*") + )// + .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("get_alias", limitedTo(alias_bwx)); + + static TestSecurityConfig.User LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( + "limited_user_B_hidden_manage_index_alias" + )// + .description("index_b*, index_hidden*, alias_bwx* with manage privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*", "index_hidden*")// + .indexPermissions("write") + .on("index_bw*", "index_hidden*")// + .indexPermissions("manage") + .on("index_bw*", "index_hidden*")// + .indexPermissions("manage_aliases") + .on("alias_bwx*") + )// + .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// + .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// + .indexMatcher("get_alias", limitedTo(alias_bwx)); + + static TestSecurityConfig.User LIMITED_USER_B_SYSTEM_INDEX_MANAGE = new TestSecurityConfig.User("limited_user_B_system_index_manage")// + .description("index_b*, .system_index_plugin with manage privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*", "system:admin/system_index") + .on("index_b*", "index_hidden*", ".system_index_plugin")// + .indexPermissions("write", "system:admin/system_index") + .on("index_bw*", ".system_index_plugin", ".system_index_plugin_*")// + .indexPermissions("manage", "system:admin/system_index") + .on("index_bw*", ".system_index_plugin", ".system_index_plugin_*") + )// + .indexMatcher( + "read", + limitedTo( + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + system_index_plugin, + system_index_plugin_not_existing + ) + )// + .indexMatcher( + "write", + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + )// + .indexMatcher( + "create_index", + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + )// + .indexMatcher( + "manage_index", + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + )// + .indexMatcher( + "manage_alias", + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + )// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// + .description("index_c*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh") + .on("index_c*")// + .indexPermissions("write") + .on("index_cw*") + )// + .indexMatcher("read", limitedTo(index_cr1, index_cw1))// + .indexMatcher("write", limitedTo(index_cw1))// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_AB1_ALIAS = new TestSecurityConfig.User("limited_user_alias_AB1")// + .description("alias_ab1")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get") + .on("alias_ab1r")// + .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get", "write", "indices:admin/refresh*") + .on("alias_ab1w*") + )// + .indexMatcher( + "read", + limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1, alias_ab1r, alias_ab1w, alias_ab1w_nowriteindex) + )// + .indexMatcher("write", limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w, alias_ab1w_nowriteindex))// + .indexMatcher("create_index", limitedTo(index_aw1, index_aw2, index_bw1))// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1, alias_ab1r, alias_ab1w)); + + static TestSecurityConfig.User LIMITED_USER_AB1_ALIAS_READ_ONLY = new TestSecurityConfig.User("limited_user_alias_AB1_read_only")// + .description("read/only on alias_ab1w, but with write privs in write index index_aw1")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "write", "indices:admin/refresh") + .on("index_aw1")// + .indexPermissions("read") + .on("alias_ab1w") + )// + .indexMatcher("read", limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w))// + .indexMatcher("write", limitedTo(index_aw1))// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_ALIAS_C1 = new TestSecurityConfig.User("limited_user_alias_C1")// + .description("alias_c1")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "write", "indices_monitor") + .on("alias_c1") + )// + .indexMatcher("read", limitedTo(index_cr1, index_cw1, alias_c1))// + .indexMatcher("write", limitedTo(index_cr1, index_cw1, alias_c1)) // + .indexMatcher("create_index", limitedTo(index_cw1))// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedTo(alias_c1)); + + static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// + .description("read/only on *")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("*") + )// + .indexMatcher("read", unlimited())// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// + .description("read/only on index_a*")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("index_a*") + )// + .indexMatcher("read", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// + .description("no privileges for existing indices")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("crud", "indices_monitor") + .on("index_does_not_exist_*") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedToNone())// + .indexMatcher("manage_alias", limitedToNone())// + .indexMatcher("get_alias", limitedToNone()); + + static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("*") + .on("*")// + .indexPermissions("*") + .on("*") + + )// + .indexMatcher( + "read", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_bwx, + alias_ab1w_nowriteindex, + index_hidden + ) + )// + .indexMatcher( + "write", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_bwx, + alias_ab1w_nowriteindex, + index_hidden + ) + )// + .indexMatcher( + "create_index", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_bwx, + alias_ab1w_nowriteindex, + index_hidden + ) + )// + .indexMatcher( + "manage_index", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_bwx, + alias_ab1w_nowriteindex, + index_hidden + ) + )// + .indexMatcher( + "manage_alias", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_bwx, + alias_ab1w_nowriteindex, + index_hidden + ) + )// + .indexMatcher( + "get_alias", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_bwx, + alias_ab1w_nowriteindex, + index_hidden + ) + ); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("write", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("create_index", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("manage_index", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("manage_alias", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); + + static List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B_CREATE_INDEX, + LIMITED_USER_B_MANAGE_INDEX, + LIMITED_USER_B_MANAGE_INDEX_ALIAS, + LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS, + LIMITED_USER_B_SYSTEM_INDEX_MANAGE, + LIMITED_USER_C, + LIMITED_USER_AB1_ALIAS, + LIMITED_USER_AB1_ALIAS_READ_ONLY, + LIMITED_USER_ALIAS_C1, + LIMITED_READ_ONLY_ALL, + LIMITED_READ_ONLY_A, + LIMITED_USER_OTHER_PRIVILEGES, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indices( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_cr1, + index_cw1, + index_hidden, + system_index_plugin + )// + .aliases(alias_ab1r, alias_ab1w, alias_ab1w_nowriteindex, alias_c1)// + .nodeSettings(Map.of("action.destructive_requires_name", false)) + .plugin(IndexAuthorizationReadOnlyIntTests.SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void putDocument() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.put("index_bw1/_doc/put_test_1", json("a", 1)); + assertThat( + httpResponse, + containsExactly(index_bw1).at("_index").reducedBy(user.indexMatcher("write")).whenEmpty(isForbidden()) + ); + } finally { + delete("index_bw1/_doc/put_test_1"); + } + } + + @Test + public void putDocument_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.put(".system_index_plugin/_doc/put_test_1", json("a", 1)); + if (clusterConfig.systemIndexPrivilegeEnabled && user.indexMatcher("write").covers(system_index_plugin)) { + assertThat(httpResponse, isCreated()); + } else if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isCreated()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(".system_index_plugin/_doc/put_test_1"); + } + } + + @Test + public void deleteDocument() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user); TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + + // Initialization + { + HttpResponse httpResponse = adminRestClient.put("index_bw1/_doc/put_delete_test_1?refresh=true", json("a", 1)); + assertThat(httpResponse, isCreated()); + } + + HttpResponse httpResponse = restClient.delete("index_bw1/_doc/put_delete_test_1"); + assertThat( + httpResponse, + containsExactly(index_bw1).at("_index").reducedBy(user.indexMatcher("write")).whenEmpty(isForbidden()) + ); + } finally { + delete("index_bw1/_doc/put_delete_test_1"); + } + } + + @Test + public void deleteByQuery_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user); TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + + HttpResponse httpResponse = adminRestClient.put( + "index_bw1/_doc/put_delete_delete_by_query_b1?refresh=true", + json("delete_by_query_test", "yes") + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "index_bw1/_doc/put_delete_delete_by_query_b2?refresh=true", + json("delete_by_query_test", "no") + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "index_aw1/_doc/put_delete_delete_by_query_a1?refresh=true", + json("delete_by_query_test", "yes") + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "index_aw1/_doc/put_delete_delete_by_query_a2?refresh=true", + json("delete_by_query_test", "no") + ); + assertThat(httpResponse, isCreated()); + + httpResponse = restClient.postJson("index_aw*,index_bw*/_delete_by_query?refresh=true&wait_for_completion=true", """ + { + "query": { + "term": { + "delete_by_query_test": "yes" + } + } + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + // dnfof is not applicable to indices:data/write/delete/byquery, so we need privileges for all indices + if (user.indexMatcher("write").coversAll(index_aw1, index_aw2, index_bw1, index_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + + } finally { + delete( + "index_bw1/_doc/put_delete_delete_by_query_b1", + "index_bw1/_doc/put_delete_delete_by_query_b2", + "index_aw1/_doc/put_delete_delete_by_query_a1", + "index_aw1/_doc/put_delete_delete_by_query_a2" + ); + } + } + + @Test + public void putDocument_bulk() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + IndexApiResponseMatchers.IndexMatcher writePrivileges = user.indexMatcher("write"); + + HttpResponse httpResponse = restClient.putJson("_bulk?refresh=true", """ + {"index": {"_index": "index_aw1", "_id": "new_doc_aw1"}} + {"a": 1} + {"index": {"_index": "index_bw1", "_id": "new_doc_bw1"}} + {"a": 1} + {"index": {"_index": "index_cw1", "_id": "new_doc_cw1"}} + {"a": 1} + """); + if (user != LIMITED_USER_NONE) { + assertThat( + httpResponse, + containsExactly(index_aw1, index_bw1, index_cw1).at("items[*].index[?(@.result == 'created')]._index") + .reducedBy(writePrivileges) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete("index_aw1/_doc/new_doc_aw1", "index_bw1/_doc/new_doc_bw1", "index_cw1/_doc/new_doc_cw1"); + } + } + + @Test + public void putDocument_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("alias_ab1w/_doc/put_doc_alias_test_1", json("a, 1")); + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.indexMatcher("write").coversAll(index_aw1, index_aw2, index_bw1)) { + assertThat(httpResponse, isCreated()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete("alias_ab1w/_doc/put_doc_alias_test_1"); + } + } + + @Test + public void putDocument_alias_noWriteIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("alias_ab1w_nowriteindex/_doc/put_doc_alias_test_1", json("a, 1")); + + if (containsExactly(alias_ab1w_nowriteindex).reducedBy(user.indexMatcher("write")).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isBadRequest()); + } + } + } + + @Test + public void putDocument_bulk_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("_bulk?refresh=true", """ + {"index": {"_index": "alias_ab1w", "_id": "put_doc_alias_bulk_test_1"}} + {"a": 1} + """); + + if (user == LIMITED_USER_A || user == LIMITED_USER_AB1_ALIAS_READ_ONLY) { + // Theoretically, a user with privileges for index_aw* could write into alias_ab2w, as the write index is index_aw1 + // However, the index resolution code is not aware that this is a write operation; thus it resolves + // to all alias members which contain also index_bw1, for which we do not have permissions + assertThat(httpResponse, containsExactly().at("items[*].index[?(@.result == 'created')]._index")); + } else if (user != LIMITED_USER_NONE) { + assertThat( + httpResponse, + containsExactly(index_aw1).at("items[*].index[?(@.result == 'created')]._index") + .reducedBy(user.indexMatcher("write")) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + + } finally { + delete("index_aw1/_doc/put_doc_alias_bulk_test_1"); + } + } + + @Test + public void putDocument_noExistingIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("index_bwx1/_doc/put_doc_non_existing_index_test_1", json("a, 1")); + assertThat( + httpResponse, + containsExactly(index_bwx1).at("_index").reducedBy(user.indexMatcher("create_index")).whenEmpty(isForbidden()) + ); + } finally { + delete(index_bwx1); + } + } + + @Test + public void createIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("index_bwx1", "{}"); + assertThat( + httpResponse, + containsExactly(index_bwx1).at("index").reducedBy(user.indexMatcher("create_index")).whenEmpty(isForbidden()) + ); + } finally { + delete(index_bwx1); + } + } + + @Test + public void createIndex_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson(".system_index_plugin_not_existing", "{}"); + + if (user.indexMatcher("create_index").covers(system_index_plugin_not_existing)) { + assertThat(httpResponse, isOk()); + } else if (user == SUPER_UNLIMITED_USER || (user == UNLIMITED_USER && !clusterConfig.systemIndexPrivilegeEnabled)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(system_index_plugin_not_existing); + } + } + + @Test + public void deleteIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_bwx1); + + HttpResponse httpResponse = restClient.delete("index_bwx1"); + if (user.indexMatcher("manage_index").isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void deleteIndex_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(system_index_plugin_not_existing); + + HttpResponse httpResponse = restClient.delete(".system_index_plugin_not_existing"); + + if (clusterConfig.systemIndexPrivilegeEnabled && user.indexMatcher("manage_index").covers(system_index_plugin_not_existing)) { + assertThat(httpResponse, isOk()); + } else if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(system_index_plugin_not_existing); + } + } + + @Test + public void createIndex_withAlias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("index_bwx1", """ + { + "aliases": { + "alias_bwx": {} + } + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.indexMatcher("manage_alias").covers(index_bwx1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void deleteAlias_staticIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1)); + + HttpResponse httpResponse = restClient.delete("index_bw1/_aliases/alias_bwx"); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.indexMatcher("manage_alias").covers(index_bw1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + + } finally { + delete(alias_bwx); + } + } + + @Test + public void deleteAlias_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1)); + + HttpResponse httpResponse = restClient.delete("*/_aliases/alias_bwx"); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + // For all non super admin users, this will be rejected by SystemIndexAccessEvaluator: + // WARN SystemIndexAccessEvaluator:361 - indices:admin/aliases for '_all' indices is not allowed for a regular user + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_createAlias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "add": { "index": "index_bw1", "alias": "alias_bwx" } } + ] + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.indexMatcher("manage_alias").covers(index_bw1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_createAlias_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "add": { "indices": ["index_bw*"], "alias": "alias_bwx" } } + ] + }"""); + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.indexMatcher("manage_alias").coversAll(index_bw1, index_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_deleteAlias_staticIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1)); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove": { "index": "index_bw1", "alias": "alias_bwx" } } + ] + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.indexMatcher("manage_alias").covers(index_bw1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_deleteAlias_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1, index_bw2)); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove": { "index": "*", "alias": "alias_bwx" } } + ] + }"""); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + // For all non super admin users, this will be rejected by SystemIndexAccessEvaluator: + // WARN SystemIndexAccessEvaluator:361 - indices:admin/aliases for '_all' indices is not allowed for a regular user + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_removeIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_bwx1); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove_index": { "index": "index_bwx1" } } + ] + }"""); + + if (user.indexMatcher("manage_index").covers(index_bwx1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void reindex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.postJson("_reindex", """ + { + "source": { "index": "index_br1" }, + "dest": { "index": "index_bwx1" } + }"""); + if (containsExactly(index_bwx1).reducedBy(user.indexMatcher("create_index")).isEmpty()) { + assertThat(httpResponse, isForbidden()); + assertThat(cluster.getAdminCertRestClient().get("index_bwx1/_search"), isNotFound()); + } else { + assertThat(httpResponse, isOk()); + assertThat(cluster.getAdminCertRestClient().get("index_bwx1/_search"), isOk()); + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void cloneIndex() throws Exception { + String sourceIndex = "index_bw1"; + String targetIndex = "index_bwx1"; + + Client client = cluster.getInternalNodeClient(); + client.admin() + .indices() + .updateSettings(new UpdateSettingsRequest(sourceIndex).settings(Settings.builder().put("index.blocks.write", true).build())) + .actionGet(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post(sourceIndex + "/_clone/" + targetIndex); + assertThat( + httpResponse, + containsExactly(index_bwx1).at("index").reducedBy(user.indexMatcher("manage_index")).whenEmpty(isForbidden()) + ); + } finally { + cluster.getInternalNodeClient() + .admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(sourceIndex).settings(Settings.builder().put("index.blocks.write", false).build()) + ) + .actionGet(); + delete(index_bwx1); + } + } + + @Test + public void closeIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("index_bw1/_close"); + assertThat( + httpResponse, + containsExactly(index_bw1).at("indices.keys()").reducedBy(user.indexMatcher("manage_index")).whenEmpty(isForbidden()) + ); + } finally { + cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("index_bw1")).actionGet(); + } + } + + @Test + public void closeIndex_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("*/_close"); + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + // For all non super admin users, this will be rejected by SystemIndexAccessEvaluator: + // WARN SystemIndexAccessEvaluator:361 - indices:admin/close for '_all' indices is not allowed for a regular user + assertThat(httpResponse, isForbidden()); + } + } finally { + cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("*")).actionGet(); + } + } + + @Test + public void closeIndex_openIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("index_bw1/_close"); + assertThat( + httpResponse, + containsExactly(index_bw1).at("indices.keys()").reducedBy(user.indexMatcher("manage_index")).whenEmpty(isForbidden()) + ); + httpResponse = restClient.post("index_bw1/_open"); + + if (containsExactly(index_bw1).reducedBy(user.indexMatcher("manage_index")).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("index_bw1")).actionGet(); + } + } + + @After + public void refresh() { + cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public IndexAuthorizationReadWriteIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(IndexAuthorizationReadWriteIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + private void createInitialTestObjects(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.createInitialTestObjects(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void delete(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.delete(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void delete(String... paths) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + for (String path : paths) { + HttpResponse response = adminRestClient.delete(path); + if (response.getStatusCode() != 200 && response.getStatusCode() != 404) { + throw new RuntimeException("Error while deleting " + path + "\n" + response.getBody()); + } + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java b/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java new file mode 100644 index 0000000000..1755ba418e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java @@ -0,0 +1,142 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableSet; + +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; +import org.opensearch.transport.client.Client; + +public class TestAlias implements TestIndexOrAliasOrDatastream { + + private final String name; + private final ImmutableSet indices; + private final TestIndexOrAliasOrDatastream writeIndex; + + private Set documentIds; + private Map documents; + + public TestAlias(String name, TestIndexOrAliasOrDatastream... indices) { + this.name = name; + this.indices = ImmutableSet.copyOf(indices); + this.writeIndex = null; + } + + TestAlias(String name, ImmutableSet indices, TestIndexOrAliasOrDatastream writeIndex) { + this.name = name; + this.indices = indices; + this.writeIndex = writeIndex; + } + + public TestAlias on(TestIndexOrAliasOrDatastream... indices) { + return new TestAlias(this.name, ImmutableSet.copyOf(indices), this.writeIndex); + } + + public TestAlias writeIndex(TestIndexOrAliasOrDatastream writeIndex) { + return new TestAlias(this.name, this.indices, writeIndex); + } + + @Override + public String toString() { + return "Test alias '" + name + "'"; + } + + @Override + public void create(Client client) { + client.admin() + .indices() + .aliases( + new IndicesAliasesRequest().addAliasAction( + IndicesAliasesRequest.AliasActions.add().indices(getIndexNamesAsArray()).alias(name) + ) + ) + .actionGet(); + + if (writeIndex != null) { + client.admin() + .indices() + .aliases( + new IndicesAliasesRequest().addAliasAction( + IndicesAliasesRequest.AliasActions.add().index(writeIndex.name()).alias(name).writeIndex(true) + ) + ) + .actionGet(); + } + } + + @Override + public void delete(Client client) { + try { + client.admin() + .indices() + .aliases(new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.remove().alias(name).indices("*"))) + .actionGet(); + } catch (AliasesNotFoundException e) { + // It is fine if the alias to be deleted does not exist + } + } + + @Override + public String name() { + return name; + } + + public ImmutableSet getIndices() { + return indices; + } + + public String[] getIndexNamesAsArray() { + return indices.stream().map(TestIndexOrAliasOrDatastream::name).collect(Collectors.toSet()).toArray(new String[0]); + } + + @Override + public Set documentIds() { + Set result = this.documentIds; + + if (result == null) { + result = new HashSet<>(); + for (TestIndexOrAliasOrDatastream testIndex : this.indices) { + result.addAll(testIndex.documentIds()); + } + + result = Collections.unmodifiableSet(result); + this.documentIds = result; + } + + return result; + } + + @Override + public Map documents() { + Map result = this.documents; + + if (result == null) { + result = new HashMap<>(); + for (TestIndexOrAliasOrDatastream testIndex : this.indices) { + result.putAll(testIndex.documents()); + } + + result = Collections.unmodifiableMap(result); + this.documents = result; + } + + return result; + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java b/src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java new file mode 100644 index 0000000000..8d3450ab5a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.util.Map; + +import com.google.common.collect.ImmutableMap; + +import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.opensearch.cluster.metadata.ComponentTemplate; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.transport.client.Client; + +public class TestComponentTemplate { + public static TestComponentTemplate DATA_STREAM_MINIMAL = new TestComponentTemplate( + "test_component_template_data_stream_minimal", + new TestMapping(new TestMapping.Property("@timestamp", "date", "date_optional_time||epoch_millis")) + ); + + private final String name; + private final TestMapping mapping; + + public TestComponentTemplate(String name, TestMapping mapping) { + this.name = name; + this.mapping = mapping; + } + + public String getName() { + return name; + } + + public TestMapping getMapping() { + return mapping; + } + + public void create(Client client) throws Exception { + try (XContentBuilder builder = JsonXContent.contentBuilder().map(getAsMap())) { + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + BytesReference.bytes(builder).streamInput() + ) + ) { + client.admin() + .indices() + .execute( + PutComponentTemplateAction.INSTANCE, + new PutComponentTemplateAction.Request(name).componentTemplate(ComponentTemplate.parse(parser)) + ) + .actionGet(); + } + } + } + + public Map getAsMap() { + return ImmutableMap.of("template", ImmutableMap.of("mappings", mapping.getAsMap())); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestData.java b/src/integrationTest/java/org/opensearch/test/framework/TestData.java index 606b56c834..5e043875d9 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestData.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestData.java @@ -11,6 +11,7 @@ package org.opensearch.test.framework; import java.nio.ByteBuffer; +import java.time.Instant; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -18,6 +19,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -32,8 +34,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.DocWriteRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.common.settings.Settings; @@ -106,16 +110,17 @@ public static TestData.Builder documentCount(int documentCount) { private Map> documentsByDepartment; private Set deletedDocuments; private long subRandomSeed; + private final String timestampColumn; - public TestData(int seed, int size, int deletedDocumentCount, int refreshAfter) { + public TestData(int seed, int size, int deletedDocumentCount, int refreshAfter, String timestampColumnName) { Random random = new Random(seed); this.ipAddresses = createRandomIpAddresses(random); this.threeWordPhrases = createRandomThreeWordPhrases(random); this.size = size; this.deletedDocumentCount = deletedDocumentCount; this.refreshAfter = refreshAfter; - // this.additionalAttributes = additionalAttributes; this.subRandomSeed = random.nextLong(); + this.timestampColumn = timestampColumnName; this.createTestDocuments(random); } @@ -129,7 +134,8 @@ private TestData( Map retainedDocuments, Map> documentsByDepartment, Set deletedDocuments, - long subRandomSeed + long subRandomSeed, + String timestampColumnName ) { super(); this.ipAddresses = ipAddresses; @@ -140,9 +146,9 @@ private TestData( this.allDocuments = allDocuments; this.retainedDocuments = retainedDocuments; this.documentsByDepartment = documentsByDepartment; - // this.additionalAttributes = additionalAttributes; this.deletedDocuments = deletedDocuments; this.subRandomSeed = subRandomSeed; + this.timestampColumn = timestampColumnName; } public void createIndex(Client client, String name, Settings settings) { @@ -157,8 +163,6 @@ public void createIndex(Client client, String name, Settings settings) { + refreshAfter ); - Random random = new Random(subRandomSeed); - long start = System.currentTimeMillis(); String mapping = """ { "_doc": { @@ -191,33 +195,60 @@ public void createIndex(Client client, String name, Settings settings) { """; client.admin().indices().create(new CreateIndexRequest(name).settings(settings).mapping(mapping)).actionGet(); - int nextRefresh = (int) Math.floor((random.nextGaussian() * 0.5 + 0.5) * refreshAfter); - int i = 0; - for (Map.Entry entry : allDocuments.entrySet()) { - String id = entry.getKey(); - TestDocument document = entry.getValue(); + this.putDocuments(client, name, -1); + } - client.index(new IndexRequest(name).source(document.content, XContentType.JSON).id(id)).actionGet(); + /** + * Writes the documents from this TestData instance to the given index. + * + * @param client the client to be used + * @param name the name of the target index + * @param rolloverAfter if this is not -1, a rollover operation will be executed for every n documents. This is useful + * for creating several generations of data stream backing indices. + */ + public void putDocuments(Client client, String name, int rolloverAfter) { + try { + Random random = new Random(subRandomSeed); + long start = System.currentTimeMillis(); + + int nextRefresh = (int) Math.floor((random.nextGaussian() * 0.5 + 0.5) * refreshAfter); + int nextRollover = rolloverAfter != -1 ? rolloverAfter : Integer.MAX_VALUE; + int i = 0; + + for (Map.Entry entry : allDocuments.entrySet()) { + String id = entry.getKey(); + TestDocument document = entry.getValue(); + + client.index( + new IndexRequest(name).source(document.content, XContentType.JSON).id(id).opType(DocWriteRequest.OpType.CREATE) + ).actionGet(); + + if (i > nextRefresh) { + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + nextRefresh = (int) Math.floor((random.nextGaussian() * 0.5 + 1) * refreshAfter) + i + 1; + } - if (i > nextRefresh) { - client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); - double g = random.nextGaussian(); + if (i > nextRollover) { + // By using rollover, we make sure that we get several generations of backing indices + client.admin().indices().rolloverIndex(new RolloverRequest(name, null)); + nextRollover += rolloverAfter; + } - nextRefresh = (int) Math.floor((g * 0.5 + 1) * refreshAfter) + i + 1; + i++; } - i++; - } + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); - client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + for (String id : deletedDocuments) { + client.delete(new DeleteRequest(name, id)).actionGet(); + } - for (String id : deletedDocuments) { - client.delete(new DeleteRequest(name, id)).actionGet(); + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + log.info("Test index creation finished after " + (System.currentTimeMillis() - start) + " ms"); + } catch (Exception e) { + throw new RuntimeException("Error while wring test documents to index " + name, e); } - - client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); - log.info("Test index creation finished after " + (System.currentTimeMillis() - start) + " ms"); } private void createTestDocuments(Random random) { @@ -365,6 +396,9 @@ private TestDocument randomDocument(Random random) { ImmutableMap.of("obj_obj_attr_text", "value_" + random.nextInt()) ) ); + if (timestampColumn != null) { + builder.put(timestampColumn, randomTimestamp(random)); + } return new TestDocument(randomId(random), builder.build()); } @@ -377,6 +411,11 @@ private String randomDepartmentName(Random random) { return departments[random.nextInt(departments.length)]; } + private String randomTimestamp(Random random) { + long epochMillis = random.longs(1, -2857691960709L, 2857691960709L).findFirst().getAsLong(); + return Instant.ofEpochMilli(epochMillis).toString(); + } + private String randomThreeWordPhrase(Random random) { return threeWordPhrases[random.nextInt(threeWordPhrases.length)]; } @@ -418,6 +457,10 @@ public int getDeletedDocumentCount() { return deletedDocumentCount; } + public Map getRetainedDocuments() { + return retainedDocuments; + } + public TestDocuments documents() { return new TestDocuments(this.retainedDocuments); } @@ -442,15 +485,16 @@ private static class Key { private final int size; private final int deletedDocumentCount; private final int refreshAfter; - // private final ImmutableMap additionalAttributes; + private final String timestampColumnName; - public Key(int seed, int size, int deletedDocumentCount, int refreshAfter) { + public Key(int seed, int size, int deletedDocumentCount, int refreshAfter, String timestampColumnName) { super(); this.seed = seed; this.size = size; this.deletedDocumentCount = deletedDocumentCount; this.refreshAfter = refreshAfter; // this.additionalAttributes = additionalAttributes; + this.timestampColumnName = timestampColumnName; } @Override @@ -461,6 +505,7 @@ public int hashCode() { result = prime * result + refreshAfter; result = prime * result + seed; result = prime * result + size; + result = prime * result + Objects.hashCode(timestampColumnName); return result; } @@ -488,6 +533,9 @@ public boolean equals(Object obj) { if (size != other.size) { return false; } + if (!Objects.equals(timestampColumnName, other.timestampColumnName)) { + return false; + } return true; } @@ -501,6 +549,7 @@ public static class Builder { private double deletedDocumentFraction = 0.06; private int refreshAfter = -1; private int segmentCount = 17; + private String timestampColumnName; public Builder() { super(); @@ -536,6 +585,11 @@ public Builder segmentCount(int segmentCount) { return this; } + public Builder timestampColumnName(String timestampColumnName) { + this.timestampColumnName = timestampColumnName; + return this; + } + public Key toKey() { if (deletedDocumentCount == -1) { this.deletedDocumentCount = (int) (this.size * deletedDocumentFraction); @@ -545,14 +599,14 @@ public Key toKey() { this.refreshAfter = this.size / this.segmentCount; } - return new Key(seed, size, deletedDocumentCount, refreshAfter); + return new Key(seed, size, deletedDocumentCount, refreshAfter, timestampColumnName); } public TestData get() { Key key = toKey(); try { - return cache.get(key, () -> new TestData(seed, size, deletedDocumentCount, refreshAfter)); + return cache.get(key, () -> new TestData(seed, size, deletedDocumentCount, refreshAfter, timestampColumnName)); } catch (ExecutionException e) { throw new RuntimeException(e); } @@ -641,6 +695,14 @@ public TestDocument get(String id) { public Set allIds() { return this.documents.keySet(); } + + public Map allDocs() { + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + for (TestDocument testDocument : this.documents.values()) { + mapBuilder.put(testDocument.id, testDocument); + } + return mapBuilder.build(); + } } public static class TestDocument { @@ -756,22 +818,6 @@ public TestDocument withOnlyAttributes(String... attributes) { return new TestDocument(this.id, ImmutableMap.copyOf(newContent)); } - /* - public TestDocument withOnlyAttributes(Set attributes) { - Map newContent = new HashMap<>(); - for (String attri) - this.content.forEach((k, v) -> { - if (k.contains(".")) { - addAttributesRecursively(this.content, newContent, k.split("\\."), 0); - } else { - if (attributes.contains(k)) { - newContent.put(k, v); - } - } - }); - - } - */ public TestDocument applyTransform(DocumentTransformer transformerFunction) { return transformerFunction.transform(this); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java b/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java new file mode 100644 index 0000000000..451c1aadf1 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.util.Map; +import java.util.Set; + +import org.opensearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.opensearch.action.admin.indices.datastream.DeleteDataStreamAction; +import org.opensearch.transport.client.Client; + +public class TestDataStream implements TestIndexOrAliasOrDatastream { + + private final String name; + private final TestData testData; + private final int rolloverAfter; + + public TestDataStream(String name, TestData testData, int rolloverAfter) { + this.name = name; + this.testData = testData; + this.rolloverAfter = rolloverAfter; + } + + @Override + public void create(Client client) { + client.admin().indices().createDataStream(new CreateDataStreamAction.Request(name)).actionGet(); + testData.putDocuments(client, name, rolloverAfter); + } + + @Override + public void delete(Client client) { + client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { name })).actionGet(); + } + + public String name() { + return name; + } + + public TestData testData() { + return testData; + } + + public static Builder name(String name) { + return new Builder().name(name); + } + + @Override + public String toString() { + return "Test data stream '" + name + '\''; + } + + public static class Builder { + private String name; + private final TestData.Builder testDataBuilder = new TestData.Builder().timestampColumnName("@timestamp") + .deletedDocumentFraction(0); + private TestData testData; + private int rolloverAfter = -1; + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder data(TestData data) { + this.testData = data; + return this; + } + + public Builder seed(int seed) { + testDataBuilder.seed(seed); + return this; + } + + public Builder documentCount(int size) { + testDataBuilder.documentCount(size); + return this; + } + + public Builder refreshAfter(int refreshAfter) { + testDataBuilder.refreshAfter(refreshAfter); + return this; + } + + public Builder rolloverAfter(int rolloverAfter) { + this.rolloverAfter = rolloverAfter; + return this; + } + + public TestDataStream build() { + if (testData == null) { + testData = testDataBuilder.get(); + } + + return new TestDataStream(name, testData, rolloverAfter); + } + } + + @Override + public Set documentIds() { + return testData().getRetainedDocuments().keySet(); + } + + @Override + public Map documents() { + return testData().getRetainedDocuments(); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java index 734e0b5333..70a4f7d102 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java @@ -28,11 +28,16 @@ package org.opensearch.test.framework; +import java.util.Map; +import java.util.Set; + import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.transport.client.Client; -public class TestIndex { +public class TestIndex implements TestIndexOrAliasOrDatastream { private final String name; private final Settings settings; @@ -44,6 +49,7 @@ public TestIndex(String name, Settings settings, TestData testData) { this.testData = testData; } + @Override public void create(Client client) { if (testData != null) { testData.createIndex(client, name, settings); @@ -52,10 +58,34 @@ public void create(Client client) { } } + @Override + public void delete(Client client) { + try { + client.admin().indices().delete(new DeleteIndexRequest(name)).actionGet(); + } catch (IndexNotFoundException e) { + // It is fine if the object to be deleted does not exist + } + } + + @Override public String name() { return name; } + @Override + public Set documentIds() { + return testData.documents().allIds(); + } + + @Override + public Map documents() { + return testData.documents().allDocs(); + } + + public TestData.TestDocument anyDocument() { + return testData.anyDocument(); + } + public static Builder name(String name) { return new Builder().name(name); } @@ -63,6 +93,7 @@ public static Builder name(String name) { public static class Builder { private String name; private Settings.Builder settings = Settings.builder(); + private TestData.Builder testDataBuilder = new TestData.Builder(); private TestData testData; public Builder name(String name) { @@ -80,15 +111,70 @@ public Builder shards(int value) { return this; } + public Builder hidden() { + settings.put("index.hidden", true); + return this; + } + public Builder data(TestData testData) { this.testData = testData; return this; } + public Builder seed(int seed) { + testDataBuilder.seed(seed); + return this; + } + + public Builder documentCount(int size) { + testDataBuilder.documentCount(size); + return this; + } + public TestIndex build() { + if (testData == null) { + testData = testDataBuilder.get(); + } + return new TestIndex(name, settings.build(), testData); } } + /** + * This returns a magic TestIndexLike object symbolizing the internal OpenSearch security + * config index. This is supposed to be used with the IndexApiResponseMatchers. + */ + public static TestIndexOrAliasOrDatastream openSearchSecurityConfigIndex() { + return OPEN_SEARCH_SECURITY_CONFIG_INDEX; + } + + private final static TestIndexOrAliasOrDatastream OPEN_SEARCH_SECURITY_CONFIG_INDEX = new TestIndexOrAliasOrDatastream() { + + @Override + public String name() { + return ".opendistro_security"; + } + + @Override + public Map documents() { + return null; + } + + @Override + public void create(Client client) { + throw new UnsupportedOperationException(); + } + + @Override + public void delete(Client client) { + throw new UnsupportedOperationException(); + } + + @Override + public Set documentIds() { + return null; + } + }; + } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java new file mode 100644 index 0000000000..f1e23f2417 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.util.Map; +import java.util.Set; + +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.transport.client.Client; + +public interface TestIndexOrAliasOrDatastream { + String name(); + + Set documentIds(); + + Map documents(); + + void create(Client client); + + void delete(Client client); + + default TestIndexOrAliasOrDatastream intersection(TestIndexOrAliasOrDatastream other) { + if (other == this) { + return this; + } + + if (!this.name().equals(other.name())) { + throw new IllegalArgumentException("Cannot intersect different indices: " + this + " vs " + other); + } + + return this; + } + + static void createInitialTestObjects(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + try (Client client = cluster.getInternalNodeClient()) { + for (TestIndexOrAliasOrDatastream testIndexOrAliasOrDatastream : testIndexOrAliasOrDatastreamArray) { + testIndexOrAliasOrDatastream.create(client); + } + } + } + + static void delete(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + try (Client client = cluster.getInternalNodeClient()) { + for (TestIndexOrAliasOrDatastream testIndexOrAliasOrDatastream : testIndexOrAliasOrDatastreamArray) { + testIndexOrAliasOrDatastream.delete(client); + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java new file mode 100644 index 0000000000..675638bb8b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.opensearch.cluster.metadata.ComposableIndexTemplate; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.transport.client.Client; + +public class TestIndexTemplate { + public static final TestIndexTemplate DATA_STREAM_MINIMAL = new TestIndexTemplate("test_index_template_data_stream_minimal", "ds_*") + .dataStream() + .composedOf(TestComponentTemplate.DATA_STREAM_MINIMAL); + + private final String name; + private final ImmutableList indexPatterns; + private Object dataStream; + private ImmutableList composedOf = ImmutableList.of(); + private int priority = 0; + + public TestIndexTemplate(String name, String... indexPatterns) { + this.name = name; + this.indexPatterns = ImmutableList.copyOf(indexPatterns); + } + + public TestIndexTemplate dataStream() { + this.dataStream = ImmutableMap.of(); + return this; + } + + public TestIndexTemplate dataStream(String k, Object v) { + this.dataStream = ImmutableMap.of(k, v); + return this; + } + + public TestIndexTemplate composedOf(TestComponentTemplate... composedOf) { + this.composedOf = ImmutableList.copyOf(composedOf); + return this; + } + + public TestIndexTemplate priority(int priority) { + this.priority = priority; + return this; + } + + public String getName() { + return name; + } + + public List getComposedOf() { + return composedOf; + } + + public void create(Client client) throws Exception { + try (XContentBuilder builder = JsonXContent.contentBuilder().map(getAsMap())) { + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + BytesReference.bytes(builder).streamInput() + ) + ) { + client.admin() + .indices() + .execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request(name).indexTemplate(ComposableIndexTemplate.parse(parser)) + ) + .actionGet(); + } + } + } + + public Map getAsMap() { + return ImmutableMap.of( + "index_patterns", + indexPatterns, + "priority", + priority, + "data_stream", + dataStream, + "composed_of", + composedOf.stream().map(TestComponentTemplate::getName).collect(Collectors.toList()) + ); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestMapping.java b/src/integrationTest/java/org/opensearch/test/framework/TestMapping.java new file mode 100644 index 0000000000..ba0ec95d0a --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/TestMapping.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework; + +import java.util.Map; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestMapping { + + private final ImmutableMap properties; + + public TestMapping(Property... properties) { + this.properties = ImmutableMap.copyOf( + ImmutableList.copyOf(properties).stream().collect(ImmutableMap.toImmutableMap(Property::getName, Property::getAsMap)) + ); + } + + public Map getAsMap() { + return ImmutableMap.of("properties", this.properties); + } + + public static class Property { + final String name; + final String type; + final String format; + + public Property(String name, String type, String format) { + this.name = name; + this.type = type; + this.format = format; + } + + public String getName() { + return name; + } + + public Map getAsMap() { + return ImmutableMap.of("type", type, "format", format); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index d58070ab45..3e09475ae5 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -72,6 +72,7 @@ import org.opensearch.security.securityconf.impl.v7.RoleV7; import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; +import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; import org.opensearch.transport.client.Client; import static org.apache.http.HttpHeaders.AUTHORIZATION; @@ -325,7 +326,6 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params if (doNotFailOnForbidden != null) { xContentBuilder.field("do_not_fail_on_forbidden", doNotFailOnForbidden); } - xContentBuilder.field("authc", authcDomainMap); if (authzDomainMap.isEmpty() == false) { xContentBuilder.field("authz", authzDomainMap); @@ -459,6 +459,8 @@ public static final class User implements UserCredentialsHolder, ToXContentObjec String requestedTenant; private Map attributes = new HashMap<>(); private Map, Object> matchers = new HashMap<>(); + private Map indexMatchers = new HashMap<>(); + private boolean adminCertUser = false; private Boolean hidden = null; @@ -512,6 +514,25 @@ public User attr(String key, String value) { return this; } + /** + * Associates an IndexMatcher with this test user. The IndexMatcher can be later used as a test oracle. + * See IndexAuthorizationReadOnlyIntTests for examples. + */ + public User indexMatcher(String key, IndexApiResponseMatchers.IndexMatcher indexMatcher) { + this.indexMatchers.put(key, indexMatcher); + return this; + } + + public IndexApiResponseMatchers.IndexMatcher indexMatcher(String key) { + IndexApiResponseMatchers.IndexMatcher result = this.indexMatchers.get(key); + + if (result != null) { + return result; + } else { + throw new RuntimeException("Unknown index matcher " + key + " in user " + this.name); + } + } + public User hash(String hash) { this.hash = hash; return this; @@ -538,6 +559,20 @@ public Set getRoleNames() { return roles.stream().map(Role::getName).collect(Collectors.toSet()); } + public String getDescription() { + return description; + } + + @Override + public boolean isAdminCertUser() { + return adminCertUser; + } + + public User adminCertUser() { + this.adminCertUser = true; + return this; + } + public Object getAttribute(String attributeName) { return attributes.get(attributeName); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index c1e6fca059..b1516dea16 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -57,7 +57,11 @@ import org.opensearch.test.framework.AuthFailureListeners; import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.OnBehalfOfConfig; +import org.opensearch.test.framework.TestAlias; +import org.opensearch.test.framework.TestComponentTemplate; +import org.opensearch.test.framework.TestDataStream; import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexTemplate; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.XffConfig; @@ -98,6 +102,10 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope private final Map remotes; private volatile LocalOpenSearchCluster localOpenSearchCluster; private final List testIndices; + private final List testAliases; + private final List testDataStreams; + private final List testComponentTemplates; + private final List testIndexTemplates; private boolean loadConfigurationIntoIndex; @@ -114,6 +122,10 @@ private LocalCluster( List clusterDependencies, Map remotes, List testIndices, + List testAliases, + List testDataStreams, + List testComponentTemplates, + List testIndexTemplates, boolean loadConfigurationIntoIndex, String defaultConfigurationInitDirectory, Integer expectedNodeStartupCount @@ -131,6 +143,10 @@ private LocalCluster( this.remotes = remotes; this.clusterDependencies = clusterDependencies; this.testIndices = testIndices; + this.testAliases = testAliases; + this.testDataStreams = testDataStreams; + this.testComponentTemplates = testComponentTemplates; + this.testIndexTemplates = testIndexTemplates; this.loadConfigurationIntoIndex = loadConfigurationIntoIndex; if (StringUtils.isNoneBlank(defaultConfigurationInitDirectory)) { System.setProperty(INIT_CONFIGURATION_DIR, defaultConfigurationInitDirectory); @@ -263,11 +279,25 @@ private void start() { } try (Client client = getInternalNodeClient()) { + for (TestComponentTemplate testComponentTemplate : this.testComponentTemplates) { + testComponentTemplate.create(client); + } + for (TestIndexTemplate indexTemplate : this.testIndexTemplates) { + indexTemplate.create(client); + } + for (TestIndex index : this.testIndices) { index.create(client); } - } + for (TestDataStream dataStream : this.testDataStreams) { + dataStream.create(client); + } + + for (TestAlias alias : this.testAliases) { + alias.create(client); + } + } } catch (Exception e) { log.error("Local ES cluster start failed", e); throw new RuntimeException(e); @@ -335,6 +365,10 @@ public static class Builder { private Map remoteClusters = new HashMap<>(); private List clusterDependencies = new ArrayList<>(); private List testIndices = new ArrayList<>(); + private List testAliases = new ArrayList<>(); + private List testDataStreams = new ArrayList<>(); + private List testIndexTemplates = new ArrayList<>(); + private List testComponentTemplates = new ArrayList<>(); private ClusterManager clusterManager = ClusterManager.DEFAULT; private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); private String clusterName = "local_cluster"; @@ -472,6 +506,28 @@ public Builder indices(Collection indices) { return this; } + public Builder aliases(TestAlias... aliases) { + this.testAliases.addAll(Arrays.asList(aliases)); + return this; + } + + public Builder dataStreams(TestDataStream... dataStreams) { + this.testDataStreams.addAll(Arrays.asList(dataStreams)); + return this; + } + + public Builder indexTemplates(TestIndexTemplate... indexTemplates) { + for (TestIndexTemplate indexTemplate : indexTemplates) { + this.testIndexTemplates.add(indexTemplate); + for (TestComponentTemplate testComponentTemplate : indexTemplate.getComposedOf()) { + if (!this.testComponentTemplates.contains(testComponentTemplate)) { + this.testComponentTemplates.add(testComponentTemplate); + } + } + } + return this; + } + public Builder users(TestSecurityConfig.User... users) { return this.users(Arrays.asList(users)); } @@ -610,6 +666,10 @@ public LocalCluster build() { clusterDependencies, remoteClusters, testIndices, + testAliases, + testDataStreams, + testComponentTemplates, + testIndexTemplates, loadConfigurationIntoIndex, defaultConfigurationInitDirectory, expectedNodeStartupCount diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index 131ff65615..5c3340ff2a 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -105,9 +105,16 @@ default TestRestClient getRestClient(UserCredentialsHolder user, CertificateData } default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { + if (user.isAdminCertUser()) { + return getRestClient(getTestCertificates().getAdminCertificateData()); + } return getRestClient(user.getName(), user.getPassword(), null, headers); } + default TestRestClient getAdminCertRestClient() { + return getRestClient(getTestCertificates().getAdminCertificateData()); + } + default RestHighLevelClient getRestHighLevelClient(String username, String password, Header... headers) { return getRestHighLevelClient(new UserCredentialsHolder() { @Override @@ -298,6 +305,10 @@ public interface UserCredentialsHolder { String getName(); String getPassword(); + + default boolean isAdminCertUser() { + return false; + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index fd4acbfb76..a5c600fae8 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -59,6 +60,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.routing.HttpRoutePlanner; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.StringEntity; @@ -72,9 +74,13 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.security.DefaultObjectMapper; +import com.nimbusds.jose.shaded.gson.Gson; + import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -184,6 +190,12 @@ public HttpResponse put(String path) { return executeRequest(uriRequest); } + public HttpResponse put(String path, HttpEntity entity, Header... headers) { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse delete(String path, Header... headers) { return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); } @@ -203,6 +215,12 @@ public HttpResponse post(String path) { return executeRequest(uriRequest); } + public HttpResponse post(String path, HttpEntity entity, Header... headers) { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse patch(String path, ToXContentObject body) { return patch(path, Strings.toString(XContentType.JSON, body)); } @@ -315,12 +333,12 @@ public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, I private void verifyContentType() { final String contentType = this.getHeader(HttpHeaders.CONTENT_TYPE).getValue(); if (contentType.contains("application/json")) { - assertThat("Response body format was not json, body: " + body, body.charAt(0), equalTo('{')); + assertThat("Response body format was not json, body: " + body, body.charAt(0), anyOf(equalTo('{'), equalTo('['))); } else { assertThat( "Response body format was json, whereas content-type was " + contentType + ", body: " + body, body.charAt(0), - not(equalTo('{')) + allOf(not(equalTo('{')), not(equalTo('['))) ); } @@ -495,4 +513,14 @@ public void close() { // TODO: Is there anything to clean up here? } + public static HttpEntity json(Object... attributes) { + Map map = new HashMap<>(); + + for (int i = 0; i < attributes.length - 1; i += 2) { + map.put(attributes[i].toString(), attributes[i + 1]); + } + + return new StringEntity(new Gson().toJson(map), ContentType.APPLICATION_JSON); + } + } diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java new file mode 100644 index 0000000000..e7375c1333 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java @@ -0,0 +1,662 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.hamcrest.Description; +import org.hamcrest.DiagnosingMatcher; +import org.hamcrest.Matcher; + +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.cluster.TestRestClient; + +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; + +import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; + +/** + * This class provides Hamcrest matchers that can be used as test oracles on the HTTP responses of index REST APIs. + *

+ * On a high level, the idea behind this class is like this: + *

    + *
  • Test users can be associated with IndexMatcher instances via the TestSecurityConfig.User.indexMatcher() method. These define the maximum index space the user can operate on. There may be several index matchers per user, targeting different groups of operations.
    • + *
  • The results of REST API calls can be also associated with a maximum space of indices the operation could work on. Combined with the user specific index matcher, one can determine the intersection of the allowed indices and thus the indices that are allowed in the particular case. The matchers support JSON path expressions to extract information on indices from the HTTP response bodies. See IndexAuthorizationReadOnlyIntTests for examples.
    • + *
+ */ +public class IndexApiResponseMatchers { + + /** + * Matchers that are directly used on HTTP responses + */ + public interface OnResponseIndexMatcher extends IndexMatcher { + + /** + * Retrieves the actual indices from the HTTP response JSON body using this JSON path expression. + * If you are asserting on an HTTP response, specifying a JSON path is madatory. + */ + OnResponseIndexMatcher at(String jsonPath); + + /** + * Calculates the intersection of this index matcher and the given other index matcher. + * If this index matcher expects the indices a,b,c and the other index matcher expects b,c,d, + * the resulting matcher will expect b,c. + */ + OnResponseIndexMatcher reducedBy(IndexMatcher other); + + /** + * Asserts on a specific HTTP status code if the set of indices expected by this matcher is empty. + */ + OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode); + + /** + * Checks whether the indices of this matcher are a subset of the other index matcher. + * If that is not the case, the given HTTP error will be expected in the response on which we are asserting. + */ + OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatchers.HttpResponseMatcher statusCode); + + default IndexMatcher butForbiddenIfIncomplete(IndexMatcher other) { + return butFailIfIncomplete(other, RestMatchers.isForbidden()); + } + + /** + * Asserts that a TestRestClient.HttpResponse object refers exactly to a specific set of indices. + *

+ * Use this matcher like this: + * 

+         *     assertThat(httpResponse, containsExactly(index_a1, index_a2).at("hits.hits[*]._index"))
+         *
+ * This will verify that the HTTP response lists the indices index_a1 and index_a2 at the place specified by the JSON path query. + *

+ * It is possible to reduce the expected indices based on a test user this way: + * 

+         *     assertThat(httpResponse, containsExactly(index_a1, index_a2).at("hits.hits[*]._index").reducedBy(user.inderMatcher("search"))
+         *
+ * This will calculate the intersection of the indices specified here and of the indices specified with the user index matcher. + * The existence of exactly these indices will be asserted. + *

+ * This method has the special feature that you can also specify data streams; it will then assert that + * the backing indices of the data streams will be present in the result set. + */ + public static OnResponseIndexMatcher containsExactly(TestIndexOrAliasOrDatastream... testIndices) { + return containsExactly(Arrays.asList(testIndices)); + } + + public static OnResponseIndexMatcher containsExactly(Collection testIndices) { + Map indexNameMap = new HashMap<>(); + boolean containsOpenSearchIndices = false; + + for (TestIndexOrAliasOrDatastream testIndex : testIndices) { + if (testIndex == TestIndex.openSearchSecurityConfigIndex()) { + containsOpenSearchIndices = true; + } else { + indexNameMap.put(testIndex.name(), testIndex); + } + } + + return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchIndices); + } + } + + /** + * Matchers that are associated with TestSecurityConfig.User objects via the indexMatcher() method + */ + public interface OnUserIndexMatcher extends IndexMatcher { + + public static IndexMatcher limitedTo(TestIndexOrAliasOrDatastream... testIndices) { + return limitedTo(Arrays.asList(testIndices)); + } + + public static IndexMatcher limitedTo(Collection testIndices) { + Map indexNameMap = new HashMap<>(); + + for (TestIndexOrAliasOrDatastream testIndex : testIndices) { + indexNameMap.put(testIndex.name(), testIndex); + } + + return new LimitedToMatcher(indexNameMap); + } + + public static IndexMatcher unlimited() { + return new UnlimitedMatcher(); + } + + public static IndexMatcher unlimitedIncludingOpenSearchSecurityIndex() { + return new UnlimitedMatcher(true); + } + + public static IndexMatcher limitedToNone() { + return new LimitedToMatcher(Collections.emptyMap()); + } + } + + /** + * The returned IndexMatcher objects implement this interface. + */ + public interface IndexMatcher extends Matcher { + /** + * Checks whether this matcher expects an empty set of indices. + */ + boolean isEmpty(); + + /** + * Returns the number of indices expected by this matcher. + */ + int size(); + + boolean containsOpenSearchIndices(); + + boolean covers(TestIndexOrAliasOrDatastream testIndex); + + default boolean coversAll(TestIndexOrAliasOrDatastream... testIndices) { + return Stream.of(testIndices).allMatch(this::covers); + } + } + + static class ContainsExactlyMatcher extends AbstractIndexMatcher implements OnResponseIndexMatcher { + private static final Pattern DS_BACKING_INDEX_PATTERN = Pattern.compile("\\.ds-(.+)-[0-9]+"); + + ContainsExactlyMatcher(Map indexNameMap, boolean containsOpenSearchIndices) { + super(indexNameMap, containsOpenSearchIndices); + } + + ContainsExactlyMatcher( + Map indexNameMap, + boolean containsOpenSearchIndices, + String jsonPath, + RestMatchers.HttpResponseMatcher statusCodeWhenEmpty + ) { + super(indexNameMap, containsOpenSearchIndices, jsonPath, statusCodeWhenEmpty); + } + + @Override + public void describeTo(Description description) { + if (indexNameMap.isEmpty()) { + if (this.statusCodeWhenEmpty.statusCode() == 200) { + description.appendText("a 200 OK response with an empty result set"); + } else { + this.statusCodeWhenEmpty.describeTo(description); + description.appendText("a response with status code " + this.statusCodeWhenEmpty); + } + } else { + description.appendText( + "a 200 OK response with exactly the indices " + indexNameMap.keySet().stream().collect(Collectors.joining(", ")) + ); + } + } + + @Override + protected boolean matchesImpl(Collection collection, Description mismatchDescription, TestRestClient.HttpResponse response) { + // Flatten the collection + collection = collection.stream() + .flatMap(e -> e instanceof Collection ? ((Collection) e).stream() : Stream.of(e)) + .collect(Collectors.toSet()); + + return matchesByIndices(collection, mismatchDescription, response); + } + + protected boolean matchesByIndices( + Collection collection, + Description mismatchDescription, + TestRestClient.HttpResponse response + ) { + ImmutableSet expectedIndices = this.getExpectedIndices(); + ImmutableSet.Builder seenIndicesBuilder = ImmutableSet.builderWithExpectedSize(expectedIndices.size()); + ImmutableSet.Builder seenOpenSearchIndicesBuilder = new ImmutableSet.Builder<>(); + + for (Object object : collection) { + String index = object.toString(); + + if (containsOpenSearchIndices && (index.startsWith(".opendistro"))) { + seenOpenSearchIndicesBuilder.add(index); + } else if (index.startsWith(".ds-")) { + // We do a special treatment for data stream backing indices. We convert these to the normal data streams if expected + // indices contains these. + java.util.regex.Matcher matcher = DS_BACKING_INDEX_PATTERN.matcher(index); + + if (matcher.matches() && expectedIndices.contains(matcher.group(1))) { + seenIndicesBuilder.add(matcher.group(1)); + } else { + seenIndicesBuilder.add(index); + } + } else { + seenIndicesBuilder.add(index); + } + } + + ImmutableSet seenIndices = seenIndicesBuilder.build(); + + ImmutableSet unexpectedIndices = Sets.difference(seenIndices, expectedIndices).immutableCopy(); + ImmutableSet missingIndices = Sets.difference(expectedIndices, seenIndices).immutableCopy(); + + if (containsOpenSearchIndices && seenOpenSearchIndicesBuilder.build().size() == 0) { + missingIndices = ImmutableSet.builderWithExpectedSize(missingIndices.size() + 1) + .addAll(missingIndices) + .add(".opensearch indices") + .build(); + } + + if (unexpectedIndices.isEmpty() && missingIndices.isEmpty()) { + return true; + } else { + if (!missingIndices.isEmpty()) { + mismatchDescription.appendText("result does not contain expected indices; found: ") + .appendValue(seenIndices) + .appendText("; missing: ") + .appendValue(missingIndices) + .appendText("\n\n") + .appendText(formatResponse(response)); + } + + if (!unexpectedIndices.isEmpty()) { + mismatchDescription.appendText("result does contain indices that were not expected: ") + .appendValue(unexpectedIndices) + .appendText("\n\n") + .appendText(formatResponse(response)); + } + return false; + } + } + + @Override + public OnResponseIndexMatcher reducedBy(IndexMatcher other) { + if (other instanceof LimitedToMatcher) { + return new ContainsExactlyMatcher( + testIndicesIntersection(this.indexNameMap, ((LimitedToMatcher) other).indexNameMap), // + this.containsOpenSearchIndices && other.containsOpenSearchIndices(), // + this.jsonPath, + this.statusCodeWhenEmpty + ); + } else if (other instanceof ContainsExactlyMatcher) { + return new ContainsExactlyMatcher( + testIndicesIntersection(this.indexNameMap, ((ContainsExactlyMatcher) other).indexNameMap), // + this.containsOpenSearchIndices && other.containsOpenSearchIndices(), // + this.jsonPath, + this.statusCodeWhenEmpty + ); + } else if (other instanceof UnlimitedMatcher) { + return new ContainsExactlyMatcher( + this.indexNameMap, // + this.containsOpenSearchIndices && other.containsOpenSearchIndices(), // + this.jsonPath, + this.statusCodeWhenEmpty + ); + } else { + throw new RuntimeException("Unexpected argument " + other); + } + } + + @Override + public OnResponseIndexMatcher at(String jsonPath) { + return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchIndices, jsonPath, statusCodeWhenEmpty); + } + + @Override + public OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode) { + return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchIndices, jsonPath, statusCode); + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return indexNameMap.containsKey(testIndex.name()); + } + + @Override + public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatchers.HttpResponseMatcher statusCode) { + if (other instanceof UnlimitedMatcher) { + return this; + } + + HashMap unmatched = new HashMap<>(this.indexNameMap); + unmatched.keySet().removeAll(((AbstractIndexMatcher) other).indexNameMap.keySet()); + + if (!unmatched.isEmpty()) { + return new StatusCodeMatcher(statusCode); + } else { + return this.reducedBy(other); + } + } + } + + static class StatusCodeMatcher extends DiagnosingMatcher implements OnResponseIndexMatcher { + private RestMatchers.HttpResponseMatcher expectedStatusCode; + + public StatusCodeMatcher(RestMatchers.HttpResponseMatcher expectedStatusCode) { + this.expectedStatusCode = expectedStatusCode; + } + + @Override + public void describeTo(Description description) { + this.expectedStatusCode.describeTo(description); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + return this.expectedStatusCode.matches(item, mismatchDescription); + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public boolean containsOpenSearchIndices() { + return true; + } + + @Override + public int size() { + return 0; + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return false; + } + + @Override + public OnResponseIndexMatcher at(String jsonPath) { + return this; + } + + @Override + public OnResponseIndexMatcher reducedBy(IndexMatcher other) { + return this; + } + + @Override + public OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode) { + return this; + } + + @Override + public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatchers.HttpResponseMatcher statusCode) { + return this; + } + } + + static class LimitedToMatcher extends AbstractIndexMatcher implements OnUserIndexMatcher { + + LimitedToMatcher(Map indexNameMap) { + super(indexNameMap, false); + } + + @Override + public void describeTo(Description description) { + if (indexNameMap.isEmpty()) { + if (this.statusCodeWhenEmpty.statusCode() == 200) { + description.appendText("a 200 OK response with an empty result set"); + } else { + this.statusCodeWhenEmpty.describeTo(description); + } + } else { + description.appendText( + "a 200 OK response no indices other than " + indexNameMap.keySet().stream().collect(Collectors.joining(", ")) + ); + } + } + + @Override + protected boolean matchesImpl(Collection collection, Description mismatchDescription, TestRestClient.HttpResponse response) { + return matchesByIndices(collection, mismatchDescription, response); + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return indexNameMap.containsKey(testIndex.name()); + } + + protected boolean matchesByIndices( + Collection collection, + Description mismatchDescription, + TestRestClient.HttpResponse response + ) { + ImmutableSet expectedIndices = this.getExpectedIndices(); + ImmutableSet.Builder seenIndicesBuilder = ImmutableSet.builderWithExpectedSize(expectedIndices.size()); + + for (Object object : collection) { + seenIndicesBuilder.add(object.toString()); + } + + ImmutableSet seenIndices = seenIndicesBuilder.build(); + ImmutableSet unexpectedIndices = Sets.difference(seenIndices, expectedIndices).immutableCopy(); + + if (unexpectedIndices.isEmpty()) { + return true; + } else { + mismatchDescription.appendText("result does contain indices that were not expected: ") + .appendValue(unexpectedIndices) + .appendText("\n\n") + .appendValue(formatResponse(response)); + return false; + } + } + } + + static class UnlimitedMatcher extends DiagnosingMatcher implements OnUserIndexMatcher { + + private final boolean containsOpenSearchIndices; + + UnlimitedMatcher() { + this.containsOpenSearchIndices = false; + } + + UnlimitedMatcher(boolean containsOpenSearchIndices) { + this.containsOpenSearchIndices = containsOpenSearchIndices; + } + + @Override + public void describeTo(Description description) { + description.appendText("unlimited indices"); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (item instanceof TestRestClient.HttpResponse) { + TestRestClient.HttpResponse response = (TestRestClient.HttpResponse) item; + + if (response.getStatusCode() != 200) { + mismatchDescription.appendText("Expected status code 200 but status was: ") + .appendValue(response.getStatusCode() + " " + response.getStatusReason()); + return false; + } + } + + return true; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public boolean containsOpenSearchIndices() { + return containsOpenSearchIndices; + } + + @Override + public int size() { + throw new IllegalStateException("The UnlimitedMatcher cannot specify a size"); + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return true; + } + } + + static abstract class AbstractIndexMatcher extends DiagnosingMatcher implements IndexMatcher { + protected final Map indexNameMap; + protected final String jsonPath; + protected final RestMatchers.HttpResponseMatcher statusCodeWhenEmpty; + protected final boolean containsOpenSearchIndices; + + AbstractIndexMatcher(Map indexNameMap, boolean containsOpenSearchIndices) { + this.indexNameMap = indexNameMap; + this.jsonPath = null; + this.statusCodeWhenEmpty = RestMatchers.isOk(); + this.containsOpenSearchIndices = containsOpenSearchIndices; + } + + AbstractIndexMatcher( + Map indexNameMap, + boolean containsOpenSearchIndices, + String jsonPath, + RestMatchers.HttpResponseMatcher statusCodeWhenEmpty + ) { + this.indexNameMap = indexNameMap; + this.jsonPath = jsonPath; + this.statusCodeWhenEmpty = statusCodeWhenEmpty; + this.containsOpenSearchIndices = containsOpenSearchIndices; + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + TestRestClient.HttpResponse response = null; + + if (item instanceof TestRestClient.HttpResponse) { + response = (TestRestClient.HttpResponse) item; + + if (indexNameMap.isEmpty()) { + if (response.getStatusCode() != this.statusCodeWhenEmpty.statusCode()) { + mismatchDescription.appendText("Status was: ") + .appendValue(response.getStatusCode() + " " + response.getStatusReason()) + .appendText("\n\n") + .appendText(formatResponse(response)); + return false; + } + + if (response.getStatusCode() != 200) { + return true; + } + } + + try { + if (response.getBody().startsWith(START_ARRAY.asString())) { + item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), List.class); + } else { + item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), Map.class); + } + } catch (JsonProcessingException e) { + mismatchDescription.appendText("Unable to parse body: ").appendValue(e.getMessage()); + return false; + } + } + + if (jsonPath != null) { + Configuration config = Configuration.builder() + .jsonProvider(new JacksonJsonProvider()) + .options(Option.SUPPRESS_EXCEPTIONS) + .build(); + + item = JsonPath.using(config).parse(item).read(jsonPath); + + if (item == null) { + mismatchDescription.appendText("Unable to find JSON Path: ") + .appendValue(jsonPath) + .appendText("\n\n") + .appendText(formatResponse(response)); + return false; + } + } + + if (!(item instanceof Collection)) { + item = Collections.singleton(item); + } + + return matchesImpl((Collection) item, mismatchDescription, response); + } + + protected abstract boolean matchesImpl( + Collection collection, + Description mismatchDescription, + TestRestClient.HttpResponse response + ); + + @Override + public boolean isEmpty() { + return indexNameMap.isEmpty(); + } + + @Override + public int size() { + if (!containsOpenSearchIndices) { + return indexNameMap.size(); + } else { + throw new RuntimeException("Size cannot be exactly specified because containsOpenSearchIndices is true"); + } + } + + @Override + public boolean containsOpenSearchIndices() { + return containsOpenSearchIndices; + } + + protected Map testIndicesIntersection( + Map map1, + Map map2 + ) { + Map result = new HashMap<>(); + + for (Map.Entry entry : map1.entrySet()) { + String key = entry.getKey(); + TestIndexOrAliasOrDatastream index1 = entry.getValue(); + TestIndexOrAliasOrDatastream index2 = map2.get(key); + + if (index2 == null) { + continue; + } + + result.put(key, index1.intersection(index2)); + } + + return Collections.unmodifiableMap(result); + } + + protected ImmutableSet getExpectedIndices() { + return ImmutableSet.copyOf(indexNameMap.keySet()); + } + + } + + private static String formatResponse(TestRestClient.HttpResponse response) { + if (response == null) { + return ""; + } + + String start = response.getStatusCode() + " " + response.getStatusReason() + "\n"; + + if (response.isJsonContentType()) { + return start + response.bodyAsJsonNode().toPrettyString(); + } else { + return start + response.getBody(); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java index 301f81b80e..96faab57c1 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java @@ -9,234 +9,172 @@ */ package org.opensearch.test.framework.matcher; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; import org.hamcrest.Description; import org.hamcrest.DiagnosingMatcher; -import org.opensearch.core.rest.RestStatus; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; public class RestMatchers { private RestMatchers() {} - public static DiagnosingMatcher isOk() { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 200 OK"); - } - - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } - - HttpResponse response = (HttpResponse) item; - - if (response.getStatusCode() == RestStatus.OK.getStatus()) { - return true; - } else { - mismatchDescription.appendText("Status is not 200 OK: ").appendValue(item); - return false; - } + public static HttpResponseMatcher isOk() { + return new HttpResponseMatcher(200, "OK"); + } - } + public static HttpResponseMatcher isCreated() { + return new HttpResponseMatcher(201, "Created"); + } - }; + public static OpenSearchErrorHttpResponseMatcher isForbidden() { + return new OpenSearchErrorHttpResponseMatcher(403, "Forbidden"); } public static DiagnosingMatcher isForbidden(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 403 Forbidden with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); - } - - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } - - HttpResponse response = (HttpResponse) item; + return isForbidden().withAttribute(jsonPointer, patternString); + } - if (response.getStatusCode() != RestStatus.FORBIDDEN.getStatus()) { - mismatchDescription.appendText("Status is not 403 Forbidden: ").appendText("\n").appendValue(item); - return false; - } + public static OpenSearchErrorHttpResponseMatcher isBadRequest() { + return new OpenSearchErrorHttpResponseMatcher(400, "Bad Request"); + } - try { - String value = response.getTextFromJsonBody(jsonPointer); + public static DiagnosingMatcher isBadRequest(String jsonPointer, String patternString) { + return isBadRequest().withAttribute(jsonPointer, patternString); + } - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } + public static OpenSearchErrorHttpResponseMatcher isNotImplemented() { + return new OpenSearchErrorHttpResponseMatcher(501, "Not Implemented"); + } - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } - } catch (Exception e) { - mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); - return false; - } - } - }; + public static DiagnosingMatcher isMethodNotImplemented(String jsonPointer, String patternString) { + return isNotImplemented().withAttribute(jsonPointer, patternString); } - public static DiagnosingMatcher isBadRequest(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 400 Bad Request with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); - } + public static OpenSearchErrorHttpResponseMatcher isInternalServerError() { + return new OpenSearchErrorHttpResponseMatcher(500, "Internal Server Error"); + } - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } + public static DiagnosingMatcher isInternalServerError(String jsonPointer, String patternString) { + return isInternalServerError().withAttribute(jsonPointer, patternString); + } - HttpResponse response = (HttpResponse) item; + public static OpenSearchErrorHttpResponseMatcher isNotFound() { + return new OpenSearchErrorHttpResponseMatcher(404, "Not Found"); + } - if (response.getStatusCode() != RestStatus.BAD_REQUEST.getStatus()) { - mismatchDescription.appendText("Status is not 400 Bad Request: ").appendText("\n").appendValue(item); - return false; - } + public static class HttpResponseMatcher extends DiagnosingMatcher { + final int statusCode; + final String statusName; + + HttpResponseMatcher(int statusCode, String statusName) { + this.statusCode = statusCode; + this.statusName = statusName; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response has status " + statusCode + " " + statusName); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!(item instanceof HttpResponse response)) { + mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); + return false; + } - try { - String value = response.getTextFromJsonBody(jsonPointer); + if (response.getStatusCode() == this.statusCode) { + return true; + } else { + mismatchDescription.appendText("Status is not " + statusCode + " " + statusName + ":\n").appendValue(item); + return false; + } + } - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } + public int statusCode() { + return this.statusCode; + } - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } - } catch (Exception e) { - mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); - return false; - } - } - }; } - public static DiagnosingMatcher isMethodNotImplemented(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 501 Method Not Implemented with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); + public static class OpenSearchErrorHttpResponseMatcher extends HttpResponseMatcher { + final ImmutableMap attributes; + + OpenSearchErrorHttpResponseMatcher(int statusCode, String statusName) { + super(statusCode, statusName); + this.attributes = ImmutableMap.of(); + } + + OpenSearchErrorHttpResponseMatcher(int statusCode, String statusName, ImmutableMap attributes) { + super(statusCode, statusName); + this.attributes = attributes; + } + + public OpenSearchErrorHttpResponseMatcher withReason(String reason) { + return withAttribute("/error/reason", reason); + } + + public OpenSearchErrorHttpResponseMatcher withType(String type) { + return withAttribute("/error/type", type); + } + + public OpenSearchErrorHttpResponseMatcher withAttribute(String jsonPointer, String value) { + return new OpenSearchErrorHttpResponseMatcher( + this.statusCode, + this.statusName, + ImmutableMap.builder().putAll(this.attributes).put(jsonPointer, value).build() + ); + } + + @Override + public void describeTo(Description description) { + super.describeTo(description); + for (Map.Entry entry : this.attributes.entrySet()) { + description.appendText(" with " + entry.getKey() + " " + entry.getValue()); } + } - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!super.matches(item, mismatchDescription)) { + return false; + } - HttpResponse response = (HttpResponse) item; + HttpResponse response = (HttpResponse) item; + boolean result = true; - if (response.getStatusCode() != RestStatus.NOT_IMPLEMENTED.getStatus()) { - mismatchDescription.appendText("Status is not 501 Method Not Implemented: ").appendText("\n").appendValue(item); - return false; - } + if (!this.attributes.isEmpty()) { + JsonNode responseDocument; try { - String value = response.getTextFromJsonBody(jsonPointer); - - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } - - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } + responseDocument = response.bodyAsJsonNode(); } catch (Exception e) { mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); return false; } - } - }; - } - - public static DiagnosingMatcher isInternalServerError(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 500 Internal Server Error with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); - } - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } - - HttpResponse response = (HttpResponse) item; - - if (response.getStatusCode() != RestStatus.INTERNAL_SERVER_ERROR.getStatus()) { - mismatchDescription.appendText("Status is not 500 Internal Server Error: ").appendText("\n").appendValue(item); - return false; + for (Map.Entry entry : this.attributes.entrySet()) { + String actualValue = responseDocument.at(entry.getKey()).asText(); + String expectedValue = entry.getValue(); + if (actualValue == null || !actualValue.contains(entry.getValue())) { + mismatchDescription.appendText(entry.getKey() + " is not " + expectedValue + ": ") + .appendValue(actualValue) + .appendText("\n"); + result = false; + } } + } - try { - String value = response.getTextFromJsonBody(jsonPointer); + if (!result) { + mismatchDescription.appendValue(item); + } - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } + return result; + } - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } - } catch (Exception e) { - mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); - return false; - } - } - }; } } From 95d2fdcb282b3fa0328fb53be74c0dea3623f352 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 16 Sep 2025 09:39:11 +0200 Subject: [PATCH 02/22] Fixes Signed-off-by: Nils Bandener --- .../int_tests/DataStreamAuthorizationReadOnlyIntTests.java | 2 +- .../int_tests/IndexAuthorizationReadOnlyIntTests.java | 1 - .../int_tests/IndexAuthorizationReadWriteIntTests.java | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index d51a5fc3b6..625053047d 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -31,7 +31,7 @@ import org.opensearch.test.framework.cluster.TestRestClient; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.is; import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 866926d741..37701e20f7 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -1228,7 +1228,6 @@ public void index_stats_pattern() throws Exception { public void getAlias_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_alias"); - System.out.println(httpResponse.getBody()); if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { // The legacy privilege evaluation also allows regular users access to metadata of the security index // This is not a security issue, as the metadata are not really security relevant diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index a1e779b933..966a0a1026 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -47,8 +47,11 @@ import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; -import static org.opensearch.test.framework.matcher.RestMatchers.*; +import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; /** * This class defines a huge test matrix for index related access controls. This class is especially for read/write operations on indices and aliases. From ec898fad53b3d0f17d960ea6ecf2df2e08573329 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 16 Sep 2025 09:55:56 +0200 Subject: [PATCH 03/22] Fixed JSON path dependency Signed-off-by: Nils Bandener --- build.gradle | 1 - .../test/framework/matcher/IndexApiResponseMatchers.java | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 28317e2761..d5c998c84f 100644 --- a/build.gradle +++ b/build.gradle @@ -575,7 +575,6 @@ allprojects { integrationTestImplementation ('com.jayway.jsonpath:json-path:2.9.0') { exclude(group: 'net.minidev', module: 'json-smart') } - integrationTestImplementation 'net.minidev:json-smart:2.6.0' } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java index e7375c1333..be6633fdbc 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java @@ -37,6 +37,7 @@ import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; @@ -573,6 +574,8 @@ protected boolean matches(Object item, Description mismatchDescription) { if (jsonPath != null) { Configuration config = Configuration.builder() .jsonProvider(new JacksonJsonProvider()) + .mappingProvider(new JacksonMappingProvider()) + .evaluationListener() .options(Option.SUPPRESS_EXCEPTIONS) .build(); From 6415df34f61a63241c37b6b8660a9b535b3e1a4c Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 17 Sep 2025 05:40:24 +0200 Subject: [PATCH 04/22] Fixed path names Signed-off-by: Nils Bandener --- .../int_tests/IndexAuthorizationReadOnlyIntTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 37701e20f7..061001d340 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -370,7 +370,7 @@ public static void stopClusters() { @Test public void search_noPattern() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - TestRestClient.HttpResponse httpResponse = restClient.get("/_search?size=1000"); + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000"); if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { assertThat( @@ -394,7 +394,7 @@ public void search_noPattern() throws Exception { @Test public void search_noPattern_noWildcards() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - TestRestClient.HttpResponse httpResponse = restClient.get("/_search?size=1000&expand_wildcards=none"); + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&expand_wildcards=none"); if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option @@ -416,7 +416,7 @@ public void search_noPattern_noWildcards() throws Exception { @Test public void search_noPattern_allowNoIndicesFalse() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - TestRestClient.HttpResponse httpResponse = restClient.get("/_search?size=1000&allow_no_indices=false"); + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&allow_no_indices=false"); if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { assertThat( From c43fe8959c91780589b45503c8061ea286e56a87 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 17 Sep 2025 08:04:54 +0200 Subject: [PATCH 05/22] Introduced more tests targeting system index privilege behavior Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 133 +++++++++++++++++- 1 file changed, 126 insertions(+), 7 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 061001d340..e3c70cf770 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -212,14 +212,20 @@ public class IndexAuthorizationReadOnlyIntTests { .roles( new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") - .indexPermissions("read", "indices_monitor", "indices:admin/analyze") - .on("index_c*")// - .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "system:admin/system_index") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") + .on("index_c*", "alias_c1")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/aliases/get", + "system:admin/system_index" + ) .on(".system_index_plugin") )// .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin))// .indexMatcher("search", limitedTo(index_c1, alias_c1, system_index_plugin))// - .indexMatcher("get_alias", limitedToNone()); + .indexMatcher("get_alias", limitedTo(index_c1, alias_c1, system_index_plugin)); /** * This user has no privileges for indices that are used in this test. But they have privileges for other indices. @@ -725,6 +731,59 @@ public void search_indexPatternAndStatic_negation() throws Exception { } } + @Test + public void search_indexPattern_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*index*/_search?size=1000&expand_wildcards=all"); + + if (user == SUPER_UNLIMITED_USER) { + // The super admin sees everything + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else if (!clusterConfig.systemIndexPrivilegeEnabled) { + // Without system index privileges, the system_index_plugin will be never included + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, index_hidden, index_hidden_dot) + .at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("search")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // Things get buggy here; basically all requests fail with a 403 + if (user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // This user is supposed to have the system index privilege for the index .system_index_plugin + // However, the system index privilege evaluation code only works correct when the system index is the + // only requested index. If also non system indices are requested in the same request, it will require + // the presence of the system index privilege for all indices. As this is not the case, the request + // will be denied with a 403 error. + assertThat(httpResponse, isForbidden()); + } else { + // The other users do not have privileges for the system index. The dnfof feature promises to filter + // out indices without authorization from eligible requests. However, the SystemIndexAccessEvaluator + // is not aware of this and just denies all these requests + // See also https://github.com/opensearch-project/security/issues/5546 + assertThat(httpResponse, isForbidden()); + } + } + } + } + @Test public void search_alias() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { @@ -789,7 +848,6 @@ public void search_aliasAndIndex_ignoreUnavailable() throws Exception { public void search_nonExisting_static() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_search?size=1000"); - // TODO adapt name to match privs for some others if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { assertThat(httpResponse, isNotFound()); } else { @@ -1228,7 +1286,7 @@ public void index_stats_pattern() throws Exception { public void getAlias_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_alias"); - if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { + if (user == UNLIMITED_USER) { // The legacy privilege evaluation also allows regular users access to metadata of the security index // This is not a security issue, as the metadata are not really security relevant assertThat(httpResponse, containsExactly(ALL_INDICES).at("$.keys()")); @@ -1283,7 +1341,7 @@ public void getAlias_aliasPattern() throws Exception { assertThat(httpResponse, isOk()); assertThat(httpResponse, containsExactly(alias_ab1).at("$.*.aliases.keys()").reducedBy(user.indexMatcher("get_alias"))); assertThat(httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("$.keys()")); - } else if (user == LIMITED_USER_ALIAS_C1) { + } else if (user == LIMITED_USER_ALIAS_C1 || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { // This is also a kind of anomaly in the legacy privilege evaluation: Even though we do not have permissions // we get a 200 response with an empty result assertThat(httpResponse, isOk()); @@ -1291,7 +1349,68 @@ public void getAlias_aliasPattern() throws Exception { } else { assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/aliases/get]")); } + } + } + @Test + public void getAlias_indexPattern_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*index*/_alias?expand_wildcards=all"); + if (user == SUPER_UNLIMITED_USER) { + // The super admin sees everything + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(alias_ab1, alias_c1).at("$.*.aliases.keys()")); + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("$.keys()") + ); + } else if (!clusterConfig.systemIndexPrivilegeEnabled) { + if (user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("$.keys()") + ); + } else { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1).at("$.*.aliases.keys()") + .reducedBy(user.indexMatcher("get_alias")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$.keys()") + .reducedBy(user.indexMatcher("get_alias")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } else { + // If the system index privilege is enabled, we only get 403 errors, as SystemIndexPrivilegeEvaluator + // is not aware of dnfof; see https://github.com/opensearch-project/security/issues/5546 + assertThat(httpResponse, isForbidden()); + } } } From 85107efd8abaffa401c454af8a42502af31686c5 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 17 Sep 2025 12:50:03 +0200 Subject: [PATCH 06/22] JSON format Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index e3c70cf770..4faa68afa1 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -868,10 +868,18 @@ public void search_nonExisting_indexPattern() throws Exception { @Test public void search_termsAggregation_index() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - TestRestClient.HttpResponse httpResponse = restClient.postJson( - "/_search", - "{\"size\":0,\"aggs\":{\"indices\":{\"terms\":{\"field\":\"_index\",\"size\":1000}}}}" - ); + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_search", """ + { + "size": 0, + "aggs": { + "indices": { + "terms": { + "field": "_index", + "size": 1000 + } + } + } + }"""); assertThat( httpResponse, @@ -1516,7 +1524,7 @@ public void resolve_indexPattern() throws Exception { public void field_caps_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { - TestRestClient.HttpResponse httpResponse = restClient.get("/_field_caps?fields=*"); + TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices") @@ -1525,7 +1533,7 @@ public void field_caps_all() throws Exception { ); } else { - TestRestClient.HttpResponse httpResponse = restClient.get("/_field_caps?fields=*"); + TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); assertThat( httpResponse, containsExactly(ALL_INDICES).at("indices") From fa0a8edee13b6237ab0f7e6109c36340f45019d8 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 17 Sep 2025 14:52:36 +0200 Subject: [PATCH 07/22] Deleted leftovers Signed-off-by: Nils Bandener --- ...DataStreamAuthorizationReadOnlyIntTests.java | 17 ++++++----------- ...ataStreamAuthorizationReadWriteIntTests.java | 13 ------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index 625053047d..afdb9bb7ff 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -58,13 +58,13 @@ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class DataStreamAuthorizationReadOnlyIntTests { - static TestDataStream ds_a1 = TestDataStream.name("ds_a1").documentCount(100).rolloverAfter(10).seed(1)/*.attr("prefix", "a")*/.build(); - static TestDataStream ds_a2 = TestDataStream.name("ds_a2").documentCount(110).rolloverAfter(10).seed(2)/*.attr("prefix", "a")*/.build(); - static TestDataStream ds_a3 = TestDataStream.name("ds_a3").documentCount(120).rolloverAfter(10).seed(3)/*.attr("prefix", "a")*/.build(); + static TestDataStream ds_a1 = TestDataStream.name("ds_a1").documentCount(100).rolloverAfter(10).seed(1).build(); + static TestDataStream ds_a2 = TestDataStream.name("ds_a2").documentCount(110).rolloverAfter(10).seed(2).build(); + static TestDataStream ds_a3 = TestDataStream.name("ds_a3").documentCount(120).rolloverAfter(10).seed(3).build(); static TestDataStream ds_ax = TestDataStream.name("ds_ax").build(); // Not existing data stream - static TestDataStream ds_b1 = TestDataStream.name("ds_b1").documentCount(51).rolloverAfter(10).seed(4)/*.attr("prefix", "b")*/.build(); - static TestDataStream ds_b2 = TestDataStream.name("ds_b2").documentCount(52).rolloverAfter(10).seed(5)/*.attr("prefix", "a")*/.build(); - static TestDataStream ds_b3 = TestDataStream.name("ds_b3").documentCount(53).rolloverAfter(10).seed(6)/*.attr("prefix", "a")*/.build(); + static TestDataStream ds_b1 = TestDataStream.name("ds_b1").documentCount(51).rolloverAfter(10).seed(4).build(); + static TestDataStream ds_b2 = TestDataStream.name("ds_b2").documentCount(52).rolloverAfter(10).seed(5).build(); + static TestDataStream ds_b3 = TestDataStream.name("ds_b3").documentCount(53).rolloverAfter(10).seed(6).build(); static TestIndex index_c1 = TestIndex.name("index_c1").documentCount(5).seed(7).build(); static final List ALL_INDICES = List.of( @@ -81,7 +81,6 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("ds_a*")// .roles( - // new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// .indexPermissions( @@ -100,7 +99,6 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("ds_b*")// .roles( - // new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// .indexPermissions( @@ -119,7 +117,6 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// .description("ds_b1")// .roles( - // new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// .indexPermissions( @@ -138,7 +135,6 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// .description("no privileges for existing indices")// .roles( - // new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// .indexPermissions( @@ -157,7 +153,6 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// .roles( - // new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// .indexPermissions("*") diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java index 7d28b8c81c..05ab55d444 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java @@ -82,7 +82,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("ds_a*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -98,7 +97,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("ds_b*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -114,7 +112,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_B_READ_ONLY_A = new TestSecurityConfig.User("limited_user_B_read_only_A")// .description("ds_b*; read only on ds_a*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -136,7 +133,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_B_AUTO_PUT_ON_ALL = new TestSecurityConfig.User("limited_user_B_auto_put_on_all")// .description("ds_b* with full auto put")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -154,7 +150,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_B_CREATE_DS = new TestSecurityConfig.User("limited_user_B_create_ds")// .description("ds_b* with create ds privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -172,7 +167,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_B_MANAGE_DS = new TestSecurityConfig.User("limited_user_B_manage_ds")// .description("ds_b* with manage privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -190,7 +184,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_AB_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_AB_manage_index")// .description("ds_a*, ds_b* with manage index privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -208,7 +201,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// .description("index_c*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh") @@ -224,7 +216,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// .description("read/only on *")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read") @@ -238,7 +229,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// .description("read/only on ds_a*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read") @@ -252,7 +242,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// .description("no privileges for existing indices")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("crud", "indices_monitor") @@ -279,7 +268,6 @@ public class DataStreamAuthorizationReadWriteIntTests { )// .description("ds_a* on backing indices")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -295,7 +283,6 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("*") From 05686eaa9576d8e194c50e10b4b6869bf7d85b66 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 17 Sep 2025 14:55:39 +0200 Subject: [PATCH 08/22] Deleted leftovers Signed-off-by: Nils Bandener --- ...taStreamAuthorizationReadOnlyIntTests.java | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index afdb9bb7ff..f957c33429 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -92,9 +92,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_a*") )// - .indexMatcher("read", limitedTo(ds_a1, ds_a2, ds_a3, ds_ax))// - .indexMatcher("read_top_level", limitedTo(ds_a1, ds_a2, ds_a3))// - .indexMatcher("get_alias", limitedToNone()); + .indexMatcher("read", limitedTo(ds_a1, ds_a2, ds_a3, ds_ax)); static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("ds_b*")// @@ -110,9 +108,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_b*") )// - .indexMatcher("read", limitedTo(ds_b1, ds_b2, ds_b3))// - .indexMatcher("read_top_level", limitedTo(ds_b1, ds_b2, ds_b3))// - .indexMatcher("get_alias", limitedToNone()); + .indexMatcher("read", limitedTo(ds_b1, ds_b2, ds_b3)); static TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// .description("ds_b1")// @@ -128,9 +124,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_b1") )// - .indexMatcher("read", limitedTo(ds_b1))// - .indexMatcher("read_top_level", limitedTo(ds_b1))// - .indexMatcher("get_alias", limitedToNone()); + .indexMatcher("read", limitedTo(ds_b1)); static TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// .description("no privileges for existing indices")// @@ -146,9 +140,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_does_not_exist_*") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("read_top_level", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .indexMatcher("read", limitedToNone()); static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// @@ -158,9 +150,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { .indexPermissions("*") .on("*")// )// - .indexMatcher("read", unlimited())// - .indexMatcher("read_top_level", unlimited())// - .indexMatcher("get_alias", unlimited()); + .indexMatcher("read", unlimited()); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -169,9 +159,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// .description("super unlimited (admin cert)")// .adminCertUser()// - .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("read_top_level", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); + .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex()); static List USERS = List.of( LIMITED_USER_A, From ca52b52998241f26bd3e6698c410e65fc76dcf09 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 17 Sep 2025 15:53:19 +0200 Subject: [PATCH 09/22] Deleted leftovers Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 102 +++++++----------- 1 file changed, 37 insertions(+), 65 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 4faa68afa1..30a9cd54b2 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -130,7 +130,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("index_a*") )// .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax))// - .indexMatcher("search", limitedTo(index_a1, index_a2, index_a3, index_ax))// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// @@ -142,7 +141,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("index_b*") )// .indexMatcher("read", limitedTo(index_b1, index_b2, index_b3))// - .indexMatcher("search", limitedTo(index_b1, index_b2, index_b3))// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// @@ -154,7 +152,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("index_b1") )// .indexMatcher("read", limitedTo(index_b1))// - .indexMatcher("search", limitedTo(index_b1))// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// @@ -166,7 +163,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("index_c*") )// .indexMatcher("read", limitedTo(index_c1, alias_c1))// - .indexMatcher("search", limitedTo(index_c1, alias_c1))// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User LIMITED_USER_ALIAS_AB1 = new TestSecurityConfig.User("limited_user_alias_AB1")// @@ -178,7 +174,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("alias_ab1*") )// .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// - .indexMatcher("search", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// .indexMatcher("get_alias", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1)); static final TestSecurityConfig.User LIMITED_USER_ALIAS_C1 = new TestSecurityConfig.User("limited_user_alias_C1")// @@ -190,7 +185,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("alias_c1") )// .indexMatcher("read", limitedTo(index_c1, alias_c1))// - .indexMatcher("search", limitedTo(index_c1, alias_c1))// .indexMatcher("get_alias", limitedTo(index_c1, alias_c1)); static final TestSecurityConfig.User LIMITED_USER_A_HIDDEN = new TestSecurityConfig.User("limited_user_A_hidden")// @@ -202,7 +196,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("index_a*", "index_hidden*", ".index_hidden*") )// .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// - .indexMatcher("search", limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User LIMITED_USER_C_WITH_SYSTEM_INDICES = new TestSecurityConfig.User( @@ -224,7 +217,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on(".system_index_plugin") )// .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin))// - .indexMatcher("search", limitedTo(index_c1, alias_c1, system_index_plugin))// .indexMatcher("get_alias", limitedTo(index_c1, alias_c1, system_index_plugin)); /** @@ -242,7 +234,6 @@ public class IndexAuthorizationReadOnlyIntTests { .on("index_does_not_exist_*") )// .indexMatcher("read", limitedToNone())// - .indexMatcher("search", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// @@ -252,7 +243,6 @@ public class IndexAuthorizationReadOnlyIntTests { .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") )// .indexMatcher("read", limitedToNone())// - .indexMatcher("search", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// @@ -281,23 +271,6 @@ public class IndexAuthorizationReadOnlyIntTests { index_ax ) )// - .indexMatcher( - "search", - limitedTo( - index_a1, - index_a2, - index_a3, - index_b1, - index_b2, - index_b3, - index_c1, - alias_ab1, - alias_c1, - index_hidden, - index_hidden_dot, - index_ax - ) - )// .indexMatcher( "get_alias", limitedTo( @@ -324,7 +297,6 @@ public class IndexAuthorizationReadOnlyIntTests { .description("super unlimited (admin cert)")// .adminCertUser()// .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("search", unlimitedIncludingOpenSearchSecurityIndex())// .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); static final List USERS = ImmutableList.of( @@ -382,7 +354,7 @@ public void search_noPattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -391,7 +363,7 @@ public void search_noPattern() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -413,7 +385,7 @@ public void search_noPattern_noWildcards() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -428,7 +400,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(isForbidden()) ); } else { @@ -437,7 +409,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) ); } } @@ -452,7 +424,7 @@ public void search_all() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -461,7 +433,7 @@ public void search_all() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -483,7 +455,7 @@ public void search_all_noWildcards() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -501,7 +473,7 @@ public void search_all_includeHidden() throws Exception { ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES ).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -516,7 +488,7 @@ public void search_wildcard() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -525,7 +497,7 @@ public void search_wildcard() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -547,7 +519,7 @@ public void search_wildcard_noWildcards() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("search")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -565,7 +537,7 @@ public void search_wildcard_includeHidden() throws Exception { ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES ).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -577,7 +549,7 @@ public void search_staticIndices() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_search?size=1000"); assertThat( httpResponse, - containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("search")).whenEmpty(isForbidden()) + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) ); } } @@ -590,7 +562,7 @@ public void search_staticIndices_ignoreUnavailable() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -601,7 +573,7 @@ public void search_staticIndices_nonExisting() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_search?size=1000"); - if (containsExactly(index_ax).reducedBy(user.indexMatcher("search")).isEmpty()) { + if (containsExactly(index_ax).reducedBy(user.indexMatcher("read")).isEmpty()) { assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); } else { assertThat(httpResponse, isNotFound()); @@ -615,7 +587,7 @@ public void search_staticIndices_hidden() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_search?size=1000"); assertThat( httpResponse, - containsExactly(index_hidden).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("search")) + containsExactly(index_hidden).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("read")) ); } } @@ -627,7 +599,7 @@ public void search_staticIndices_systemIndex() throws Exception { if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { assertThat( httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("search")) + containsExactly(system_index_plugin).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("read")) ); } else { // legacy privilege evaluation without system index privilege enabled @@ -651,7 +623,7 @@ public void search_indexPattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -665,7 +637,7 @@ public void search_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -687,7 +659,7 @@ public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exce assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -708,7 +680,7 @@ public void search_indexPattern_noWildcards() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -725,7 +697,7 @@ public void search_indexPatternAndStatic_negation() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -752,7 +724,7 @@ public void search_indexPattern_includeHidden() throws Exception { index_hidden_dot, system_index_plugin ).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else if (!clusterConfig.systemIndexPrivilegeEnabled) { @@ -761,7 +733,7 @@ public void search_indexPattern_includeHidden() throws Exception { httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, index_hidden, index_hidden_dot) .at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -793,7 +765,7 @@ public void search_alias() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(isForbidden()) ); } @@ -808,7 +780,7 @@ public void search_alias_pattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -821,7 +793,7 @@ public void search_alias_pattern_negation() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -836,7 +808,7 @@ public void search_aliasAndIndex_ignoreUnavailable() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("search")) + .reducedBy(user.indexMatcher("read")) .whenEmpty(isForbidden()) ); @@ -885,7 +857,7 @@ public void search_termsAggregation_index() throws Exception { httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at( "aggregations.indices.buckets[*].key" - ).reducedBy(user.indexMatcher("search")).whenEmpty(isOk()) + ).reducedBy(user.indexMatcher("read")).whenEmpty(isOk()) ); } } @@ -904,7 +876,7 @@ public void search_pit() throws Exception { index_b3 ); - if (indexMatcher.reducedBy(user.indexMatcher("search")).isEmpty()) { + if (indexMatcher.reducedBy(user.indexMatcher("read")).isEmpty()) { assertThat( httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") @@ -920,7 +892,7 @@ public void search_pit() throws Exception { } """, pitId)); assertThat(httpResponse, isOk()); - assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("search"))); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("read"))); } } } @@ -948,7 +920,7 @@ public void search_pit_all() throws Exception { ); } - if (indexMatcher.reducedBy(user.indexMatcher("search")).isEmpty()) { + if (indexMatcher.reducedBy(user.indexMatcher("read")).isEmpty()) { assertThat( httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") @@ -971,7 +943,7 @@ public void search_pit_all() throws Exception { assertThat(httpResponse, isForbidden()); } else { assertThat(httpResponse, isOk()); - assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("search"))); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("read"))); } } } @@ -984,7 +956,7 @@ public void search_pit_static() throws Exception { IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher = containsExactly(index_a1); - if (indexMatcher.reducedBy(user.indexMatcher("search")).isEmpty()) { + if (indexMatcher.reducedBy(user.indexMatcher("read")).isEmpty()) { assertThat( httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") @@ -999,7 +971,7 @@ public void search_pit_static() throws Exception { } } """, pitId)); - assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("search"))); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("read"))); } } } @@ -1009,7 +981,7 @@ public void search_pit_wrongIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.post("index_a*/_search/point_in_time?keep_alive=1m"); - if (user.indexMatcher("search").coversAll(index_a1, index_a2, index_a3)) { + if (user.indexMatcher("read").coversAll(index_a1, index_a2, index_a3)) { assertThat(httpResponse, isOk()); String pitId = httpResponse.getTextFromJsonBody("/pit_id"); httpResponse = restClient.postJson("index_b*/_search?size=1000", String.format(""" From b1c78cbebfb424c7a0d108ab6ad4b3d91a3e248c Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 18 Sep 2025 11:28:02 +0200 Subject: [PATCH 10/22] More tests Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 177 ++++++- .../IndexAuthorizationReadWriteIntTests.java | 56 ++- .../SnapshotAuthorizationIntTests.java | 382 ++++++++++++++++ .../opensearch/test/framework/TestAlias.java | 20 +- .../opensearch/test/framework/TestData.java | 33 +- .../test/framework/cluster/LocalCluster.java | 22 + .../matcher/IndexApiResponseMatchers.java | 430 ++++++++++-------- .../resources/log4j2-test.properties | 3 + 8 files changed, 894 insertions(+), 229 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 30a9cd54b2..b2f679b5f1 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; @@ -26,6 +27,7 @@ import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SystemIndexPlugin; +import org.opensearch.script.mustache.MustacheModulePlugin; import org.opensearch.test.framework.TestAlias; import org.opensearch.test.framework.TestData; import org.opensearch.test.framework.TestIndex; @@ -35,6 +37,8 @@ import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; @@ -64,6 +68,8 @@ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class IndexAuthorizationReadOnlyIntTests { + // TODO pit_segments + static final TestIndex index_a1 = TestIndex.name("index_a1").documentCount(100).seed(1).build(); static final TestIndex index_a2 = TestIndex.name("index_a2").documentCount(110).seed(2).build(); static final TestIndex index_a3 = TestIndex.name("index_a3").documentCount(120).seed(3).build(); @@ -76,8 +82,9 @@ public class IndexAuthorizationReadOnlyIntTests { static final TestIndex index_hidden_dot = TestIndex.name(".index_hidden_dot").hidden().documentCount(1).seed(8).build(); static final TestIndex system_index_plugin = TestIndex.name(".system_index_plugin").hidden().documentCount(1).seed(8).build(); - static final TestAlias alias_ab1 = new TestAlias("alias_ab1", index_a1, index_a2, index_a3, index_b1); - static final TestAlias alias_c1 = new TestAlias("alias_c1", index_c1); + static final TestAlias alias_ab1 = new TestAlias("alias_ab1").on(index_a1, index_a2, index_a3, index_b1); + static final TestAlias alias_c1 = new TestAlias("alias_c1").on(index_c1); + static final TestAlias alias_with_system_index = new TestAlias(".alias_with_system_index").hidden().on(system_index_plugin); static final List ALL_INDICES_EXCEPT_SYSTEM_INDICES = List.of( index_a1, @@ -118,6 +125,7 @@ public class IndexAuthorizationReadOnlyIntTests { index_hidden, index_hidden_dot, system_index_plugin, + alias_with_system_index, openSearchSecurityConfigIndex() ); @@ -216,7 +224,7 @@ public class IndexAuthorizationReadOnlyIntTests { ) .on(".system_index_plugin") )// - .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin))// + .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index))// .indexMatcher("get_alias", limitedTo(index_c1, alias_c1, system_index_plugin)); /** @@ -330,8 +338,8 @@ static LocalCluster.Builder clusterBuilder() { index_hidden_dot, system_index_plugin )// - .aliases(alias_ab1, alias_c1)// - .plugin(SystemIndexTestPlugin.class); + .aliases(alias_ab1, alias_c1, alias_with_system_index)// + .plugin(SystemIndexTestPlugin.class, MustacheModulePlugin.class); } @AfterClass @@ -615,6 +623,51 @@ public void search_staticIndices_systemIndex() throws Exception { } } + @Test + public void search_staticIndices_systemIndex_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get(".alias_with_system_index/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index")); + } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { + if (user == UNLIMITED_USER || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, + // but withholds documents on the DLS level + assertThat(httpResponse, isOk()); + assertThat( + httpResponse, + containsExactly().at("hits.hits[*]._index")); + } else { + assertThat(httpResponse, isForbidden()); + } + } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { + if (user == UNLIMITED_USER) { + // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, + // but withholds documents on the DLS level + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + } + } else { + if (user.indexMatcher("read").covers(alias_with_system_index)) { + assertThat(httpResponse, isOk()); + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index")); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + } + @Test public void search_indexPattern() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { @@ -796,9 +849,66 @@ public void search_alias_pattern_negation() throws Exception { .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); +/* + if (user != LIMITED_USER_NONE) { + if (clusterConfig.systemIndexPrivilegeEnabled) { + // If the system index privilege is enabled, we might also see the system_index_plugin index (being included via the alias) + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } else { + assertThat(httpResponse, isForbidden()); + }*/ } } + @Test + public void search_alias_pattern_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*alias*/_search?size=1000&expand_wildcards=all"); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index")); + } else if (user != LIMITED_USER_NONE) { + if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { + // For all users without the system index permission, SystemIndexAccessEvaluator shuts the door + // For the user with the system index permission, that happens as well, as SystemIndexAccessEvaluator expects the permission for all requested indices, even if they are not system indices + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isOk())); + } + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test public void search_aliasAndIndex_ignoreUnavailable() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { @@ -1002,6 +1112,45 @@ public void search_pit_wrongIndex() throws Exception { } } + /** + * Moved from https://github.com/opensearch-project/security/blob/eb7153d772e9e00d49d9cb5ffafb33b5f02399fc/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java#L103 + * See also https://github.com/opensearch-project/security/issues/1678 + */ + @Test + public void search_template_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + String params = """ + { + "department": [%s] + }""".formatted(TestData.DEPARTMENTS.stream().map(s -> '"' + s + '"').collect(joining(","))); + String query = """ + { + "query": { + "terms": { + "attr_text_1": [ + "{{#department}}", + "{{.}}", + "{{/department}}" + ] + } + } + } + """; + + TestRestClient.HttpResponse httpResponse = restClient.getWithJsonBody("index_a1/_search/template?size=1000", + """ + { + "params": %s, + "source": "%s" + }""".formatted(params, escapeJson(query))); + + assertThat( + httpResponse, + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ); + } + } + @Test public void msearch_staticIndices() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { @@ -1186,13 +1335,19 @@ public void cat_aliases_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases?format=json"); - if (!user.indexMatcher("get_alias").isEmpty()) { + if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { assertThat( - httpResponse, - containsExactly(alias_ab1, alias_c1).at("$[*].alias").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isOk()) - ); + httpResponse, + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias")); } else { - assertThat(httpResponse, isForbidden()); + if (!user.indexMatcher("get_alias").isEmpty()) { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } } } } @@ -1339,7 +1494,7 @@ public void getAlias_indexPattern_includeHidden() throws Exception { if (user == SUPER_UNLIMITED_USER) { // The super admin sees everything assertThat(httpResponse, isOk()); - assertThat(httpResponse, containsExactly(alias_ab1, alias_c1).at("$.*.aliases.keys()")); + assertThat(httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()")); assertThat( httpResponse, containsExactly( diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index 966a0a1026..7c81b6c5bc 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -86,9 +86,9 @@ public class IndexAuthorizationReadWriteIntTests { .documentCount(0) .build(); // not initially created - static final TestAlias alias_ab1r = new TestAlias("alias_ab1r", index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1); - static final TestAlias alias_ab1w = new TestAlias("alias_ab1w", index_aw1, index_aw2, index_bw1).writeIndex(index_aw1); - static final TestAlias alias_ab1w_nowriteindex = new TestAlias("alias_ab1w_nowriteindex", index_aw1, index_aw2, index_bw1); + static final TestAlias alias_ab1r = new TestAlias("alias_ab1r").on(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1); + static final TestAlias alias_ab1w = new TestAlias("alias_ab1w").on(index_aw1, index_aw2, index_bw1).writeIndex(index_aw1); + static final TestAlias alias_ab1w_nowriteindex = new TestAlias("alias_ab1w_nowriteindex").on(index_aw1, index_aw2, index_bw1); static final TestAlias alias_c1 = new TestAlias("alias_c1", index_cr1, index_cw1); @@ -198,13 +198,13 @@ public class IndexAuthorizationReadWriteIntTests { static TestSecurityConfig.User LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( "limited_user_B_hidden_manage_index_alias" )// - .description("index_b*, index_hidden*, alias_bwx* with manage privs")// + .description("index_b*, index_hidden*, alias_bwx* with manage privs, index_a* read only")// .roles( // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") - .on("index_b*", "index_hidden*")// + .on("index_a*", "index_b*", "index_hidden*")// .indexPermissions("write") .on("index_bw*", "index_hidden*")// .indexPermissions("manage") @@ -212,7 +212,7 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("manage_aliases") .on("alias_bwx*") )// - .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .indexMatcher("read", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// @@ -1155,6 +1155,50 @@ public void closeIndex_openIndex() throws Exception { } } + @Test + public void rollover_explicitTargetIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1, index_bw2)); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove": { "index": "*", "alias": "alias_bwx" } } + ] + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + // This is only allowed if we have privileges for all indices, even if not all indices are member of alias_bwx + if (user.indexMatcher("manage_alias") + .coversAll( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw1, + index_cr1, + index_cw1 + )) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else { + if (user.indexMatcher("manage_alias").coversAll(alias_bwx)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + + } finally { + delete(alias_bwx); + } + } + @After public void refresh() { cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java new file mode 100644 index 0000000000..ad222f71e3 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java @@ -0,0 +1,382 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.apache.hc.core5.http.HttpEntity; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.test.framework.TestIndex; +import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; + +/** + * TODO requests on non master node + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class SnapshotAuthorizationIntTests { + static final TestIndex index_a1 = TestIndex.name("index_ar1").documentCount(10).seed(1).build(); + static final TestIndex index_a2 = TestIndex.name("index_ar2").documentCount(11).seed(2).build(); + static final TestIndex index_a3 = TestIndex.name("index_ar3").documentCount(12).seed(3).build(); + static final TestIndex index_b1 = TestIndex.name("index_br1").documentCount(4).seed(4).build(); + static final TestIndex index_b2 = TestIndex.name("index_br2").documentCount(5).seed(5).build(); + static final TestIndex index_b3 = TestIndex.name("index_br3").documentCount(6).seed(6).build(); + + static final TestIndex system_index_plugin_not_existing = TestIndex.name(".system_index_plugin_not_existing") + .hidden() + .documentCount(0) + .build(); // not initially created + + static final TestIndex index_awx1 = TestIndex.name("index_awx1").documentCount(10).seed(11).build(); // not initially created + static final TestIndex index_awx2 = TestIndex.name("index_awx2").documentCount(10).seed(12).build(); // not initially created + + static final TestIndex index_bwx1 = TestIndex.name("index_bwx1").documentCount(10).seed(13).build(); // not initially created + static final TestIndex index_bwx2 = TestIndex.name("index_bwx2").documentCount(10).seed(14).build(); // not initially created + + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("index_a*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_a*")// + .indexPermissions("write", "manage") + .on("index_aw*") + )// + .indexMatcher("read", limitedTo(index_a1, index_a2, index_awx1, index_awx2))// + .indexMatcher("write", limitedTo(index_awx1, index_awx2)); + + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("index_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write", "manage") + .on("index_bw*") + )// + .indexMatcher("read", limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_bwx1, index_bwx2)); + + static TestSecurityConfig.User LIMITED_USER_B_SYSTEM_INDEX = new TestSecurityConfig.User("limited_user_B_system_index")// + .description("index_b*, .system_index_plugin")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write", "manage") + .on("index_bw*") + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*", "system:admin/system_index") + .on(".system_index_plugin", ".system_index_plugin_not_existing") + .indexPermissions("write", "manage", "system:admin/system_index") + .on(".system_index_plugin_not_existing") + + )// + .indexMatcher("read", limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_bwx1, index_bwx2, system_index_plugin_not_existing)); + + static TestSecurityConfig.User LIMITED_USER_AB = new TestSecurityConfig.User("limited_user_AB")// + .description("index_a*, index_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_a*", "index_b*")// + .indexPermissions("write", "manage") + .on("index_aw*", "index_bw*") + )// + .indexMatcher("read", limitedTo(index_a1, index_a2, index_awx1, index_awx2, index_b1, index_b2, index_bwx1, index_bwx2))// + .indexMatcher("write", limitedTo(index_awx1, index_awx2, index_bwx1, index_bwx2)); + + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .indexMatcher("read", limitedToNone())// + .indexMatcher("write", limitedToNone()); + + static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor", "manage_snapshots") + .indexPermissions("*") + .on("*")// + + )// + .indexMatcher( + "read", + limitedTo(index_a1, index_a2, index_a3, index_awx1, index_awx2, index_b1, index_b2, index_b3, index_bwx1, index_bwx2) + )// + .indexMatcher( + "write", + limitedTo(index_a1, index_a2, index_a3, index_awx1, index_awx2, index_b1, index_b2, index_b3, index_bwx1, index_bwx2) + ); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static final TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// + .indexMatcher("write", unlimitedIncludingOpenSearchSecurityIndex()); + + static final List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B_SYSTEM_INDEX, + LIMITED_USER_AB, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indices(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3)// + .snapshotRepositories("test_repository") + .plugin(IndexAuthorizationReadOnlyIntTests.SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void restore_singleIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + delete(index_awx1); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true" + ); + + assertThat( + httpResponse, + containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) + ); + + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1); + } + } + + @Test + public void restore_singleIndex_rename1() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true", + json("rename_pattern", "index_(.+)x1", "rename_replacement", "index_$1x2") + ); + + assertThat( + httpResponse, + containsExactly(index_awx2).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) + ); + + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1, index_awx2); + } + } + + @Test + public void restore_singleIndex_rename2() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true", + json("rename_pattern", "index_a(.*)", "rename_replacement", "index_b$1") + ); + + assertThat( + httpResponse, + containsExactly(index_bwx1).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) + ); + + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1, index_bwx1); + } + } + + @Test + public void restore_singleIndex_renameToSystemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true", + json("rename_pattern", "index_awx1", "rename_replacement", system_index_plugin_not_existing.name()) + ); + + if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(system_index_plugin_not_existing).at("snapshot.indices") + .butForbiddenIfIncomplete(user.indexMatcher("write")) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1, system_index_plugin_not_existing); + } + } + + @Test + public void restore_singleIndexFromAllIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/all_index_snapshot", json()); + + delete(index_awx1); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/all_index_snapshot/_restore?wait_for_completion=true", + json("indices", "index_awx1") + ); + + assertThat( + httpResponse, + containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) + ); + + } finally { + delete("_snapshot/test_repository/all_index_snapshot"); + delete(index_awx1); + } + } + + @Test + public void restore_all_globalState() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1, index_awx2, index_bwx1, index_bwx2); + createInitialTestSnapshot("_snapshot/test_repository/all_index_snapshot", json("indices", "index_*w*")); + + delete(index_awx1, index_awx2, index_bwx1, index_bwx2); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/all_index_snapshot/_restore?wait_for_completion=true", + json("include_global_state", true) + ); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + + } finally { + delete("_snapshot/test_repository/all_index_snapshot"); + delete(index_awx1, index_awx2, index_bwx1, index_bwx2); + } + } + + @After + public void refresh() { + cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public SnapshotAuthorizationIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(SnapshotAuthorizationIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + private void createInitialTestObjects(TestIndexOrAliasOrDatastream... testIndexLikeArray) { + TestIndexOrAliasOrDatastream.createInitialTestObjects(cluster, testIndexLikeArray); + } + + private void createInitialTestSnapshot(String snapshotName, HttpEntity requestBody) { + try (TestRestClient client = cluster.getAdminCertRestClient()) { + TestRestClient.HttpResponse httpResponse = client.put(snapshotName + "?wait_for_completion=true", requestBody); + assertThat(httpResponse, isOk()); + } + } + + private void delete(TestIndexOrAliasOrDatastream... testIndexLikeArray) { + TestIndexOrAliasOrDatastream.delete(cluster, testIndexLikeArray); + } + + private void delete(String... paths) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + for (String path : paths) { + TestRestClient.HttpResponse response = adminRestClient.delete(path); + if (response.getStatusCode() != 200 && response.getStatusCode() != 404) { + throw new RuntimeException("Error while deleting " + path + "\n" + response.getBody()); + } + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java b/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java index 1755ba418e..a763a365e2 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSet; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.common.settings.Settings; import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; import org.opensearch.transport.client.Client; @@ -29,6 +30,7 @@ public class TestAlias implements TestIndexOrAliasOrDatastream { private final String name; private final ImmutableSet indices; private final TestIndexOrAliasOrDatastream writeIndex; + private final boolean hidden; private Set documentIds; private Map documents; @@ -37,20 +39,26 @@ public TestAlias(String name, TestIndexOrAliasOrDatastream... indices) { this.name = name; this.indices = ImmutableSet.copyOf(indices); this.writeIndex = null; + this.hidden = false; } - TestAlias(String name, ImmutableSet indices, TestIndexOrAliasOrDatastream writeIndex) { + TestAlias(String name, ImmutableSet indices, TestIndexOrAliasOrDatastream writeIndex, boolean hidden) { this.name = name; this.indices = indices; this.writeIndex = writeIndex; + this.hidden = hidden; } public TestAlias on(TestIndexOrAliasOrDatastream... indices) { - return new TestAlias(this.name, ImmutableSet.copyOf(indices), this.writeIndex); + return new TestAlias(this.name, ImmutableSet.copyOf(indices), this.writeIndex, this.hidden); } public TestAlias writeIndex(TestIndexOrAliasOrDatastream writeIndex) { - return new TestAlias(this.name, this.indices, writeIndex); + return new TestAlias(this.name, this.indices, writeIndex, this.hidden); + } + + public TestAlias hidden() { + return new TestAlias(this.name, this.indices, this.writeIndex, true); } @Override @@ -64,7 +72,7 @@ public void create(Client client) { .indices() .aliases( new IndicesAliasesRequest().addAliasAction( - IndicesAliasesRequest.AliasActions.add().indices(getIndexNamesAsArray()).alias(name) + IndicesAliasesRequest.AliasActions.add().indices(getIndexNamesAsArray()).alias(name).isHidden(hidden) ) ) .actionGet(); @@ -139,4 +147,8 @@ public Map documents() { return result; } + + public static TestIndex.Builder name(String name) { + return new TestIndex.Builder().name(name); + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestData.java b/src/integrationTest/java/org/opensearch/test/framework/TestData.java index 5e043875d9..7c28d7912e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestData.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestData.java @@ -29,6 +29,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; @@ -92,6 +93,8 @@ public static TestData.Builder documentCount(int documentCount) { "attr_object.obj_attr_object.obj_obj_attr_text.keyword" ); + public static final ImmutableList DEPARTMENTS = ImmutableList.of("dept_a_1", "dept_a_2", "dept_a_3", "dept_b_1", "dept_b_2", "dept_c", "dept_d"); + private static final Cache cache; static { @@ -101,7 +104,6 @@ public static TestData.Builder documentCount(int documentCount) { private String[] ipAddresses; private String[] threeWordPhrases; - private String[] departments = new String[] { "dept_a_1", "dept_a_2", "dept_a_3", "dept_b_1", "dept_b_2", "dept_c", "dept_d" }; private int size; private int deletedDocumentCount; private int refreshAfter; @@ -124,33 +126,6 @@ public TestData(int seed, int size, int deletedDocumentCount, int refreshAfter, this.createTestDocuments(random); } - private TestData( - String[] ipAddresses, - String[] departments, - int size, - int deletedDocumentCount, - int refreshAfter, - Map allDocuments, - Map retainedDocuments, - Map> documentsByDepartment, - Set deletedDocuments, - long subRandomSeed, - String timestampColumnName - ) { - super(); - this.ipAddresses = ipAddresses; - this.departments = departments; - this.size = size; - this.deletedDocumentCount = deletedDocumentCount; - this.refreshAfter = refreshAfter; - this.allDocuments = allDocuments; - this.retainedDocuments = retainedDocuments; - this.documentsByDepartment = documentsByDepartment; - this.deletedDocuments = deletedDocuments; - this.subRandomSeed = subRandomSeed; - this.timestampColumn = timestampColumnName; - } - public void createIndex(Client client, String name, Settings settings) { log.info( "creating test index " @@ -408,7 +383,7 @@ private String randomIpAddress(Random random) { } private String randomDepartmentName(Random random) { - return departments[random.nextInt(departments.length)]; + return DEPARTMENTS.get(random.nextInt(DEPARTMENTS.size())); } private String randomTimestamp(Random random) { diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index b1516dea16..a11462f3a4 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -44,6 +44,7 @@ import org.apache.logging.log4j.Logger; import org.junit.rules.ExternalResource; +import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.opensearch.common.settings.Settings; import org.opensearch.node.PluginAwareNode; import org.opensearch.plugins.Plugin; @@ -106,6 +107,7 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope private final List testDataStreams; private final List testComponentTemplates; private final List testIndexTemplates; + private final List testSnapshotRepositories; private boolean loadConfigurationIntoIndex; @@ -126,6 +128,7 @@ private LocalCluster( List testDataStreams, List testComponentTemplates, List testIndexTemplates, + List testSnapshotRepositories, boolean loadConfigurationIntoIndex, String defaultConfigurationInitDirectory, Integer expectedNodeStartupCount @@ -147,6 +150,7 @@ private LocalCluster( this.testDataStreams = testDataStreams; this.testComponentTemplates = testComponentTemplates; this.testIndexTemplates = testIndexTemplates; + this.testSnapshotRepositories = testSnapshotRepositories; this.loadConfigurationIntoIndex = loadConfigurationIntoIndex; if (StringUtils.isNoneBlank(defaultConfigurationInitDirectory)) { System.setProperty(INIT_CONFIGURATION_DIR, defaultConfigurationInitDirectory); @@ -297,6 +301,10 @@ private void start() { for (TestAlias alias : this.testAliases) { alias.create(client); } + + for (String snapshotRepository : this.testSnapshotRepositories) { + createSnapshotRepository(client, snapshotRepository); + } } } catch (Exception e) { log.error("Local ES cluster start failed", e); @@ -349,6 +357,13 @@ public void triggerConfigurationReloadForCTypes(Client client, List cType } } + private void createSnapshotRepository(Client client, String snapshotRepository) { + client.admin() + .cluster() + .putRepository(new PutRepositoryRequest(snapshotRepository).type("fs").settings(Map.of("location", getSnapshotDirPath()))) + .actionGet(); + } + public CertificateData getAdminCertificate() { return testCertificates.getAdminCertificateData(); } @@ -369,6 +384,7 @@ public static class Builder { private List testDataStreams = new ArrayList<>(); private List testIndexTemplates = new ArrayList<>(); private List testComponentTemplates = new ArrayList<>(); + private List testSnapshotRepositories = new ArrayList<>(); private ClusterManager clusterManager = ClusterManager.DEFAULT; private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); private String clusterName = "local_cluster"; @@ -528,6 +544,11 @@ public Builder indexTemplates(TestIndexTemplate... indexTemplates) { return this; } + public Builder snapshotRepositories(String... repositoryNames) { + this.testSnapshotRepositories.addAll(Arrays.asList(repositoryNames)); + return this; + } + public Builder users(TestSecurityConfig.User... users) { return this.users(Arrays.asList(users)); } @@ -670,6 +691,7 @@ public LocalCluster build() { testDataStreams, testComponentTemplates, testIndexTemplates, + testSnapshotRepositories, loadConfigurationIntoIndex, defaultConfigurationInitDirectory, expectedNodeStartupCount diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java index be6633fdbc..04769faf78 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java @@ -170,15 +170,225 @@ public interface IndexMatcher extends Matcher { */ int size(); - boolean containsOpenSearchIndices(); + /** + * Returns true if the set of indices is expected to contain the security config index + */ + boolean containsOpenSearchSecurityIndex(); + /** + * Returns true if this matcher expects the given index to be present + */ boolean covers(TestIndexOrAliasOrDatastream testIndex); + /** + * Returns true if this matcher expects all the given indices to be present + */ default boolean coversAll(TestIndexOrAliasOrDatastream... testIndices) { return Stream.of(testIndices).allMatch(this::covers); } } + // ---------------------------------------------------------------------------------- + // Actual matcher implementations + // (created by static methods in OnResponseIndexMatcher and OnUserIndexMatcher above) + // ---------------------------------------------------------------------------------- + + /** + * Base implementation for all matchers. The primary working mode of these matchers is to + * expect TestRestClient.HttpResponse objects and to extract index names from the response + * body via a jsonPath (specified with the at() method). However, the matchers will also + * work on any string collection; then, the json path is not necessary. + */ + static abstract class AbstractIndexMatcher extends DiagnosingMatcher implements IndexMatcher { + /** + * The indices expected by this matcher. + */ + protected final Map expectedIndices; + + /** + * The matcher will extract the indices from the REST response body using this JSON path expression. + */ + protected final String jsonPath; + + /** + * If the matcher expects an empty set of indices, this can actually mean two things: + *
    + *
  1. The response is expected to be successful (i.e. has a 200 OK status) and returns an empty set of indices
    2. + *
  2. The response has failed with a non 200 status code
    3. + *
+ * The expected status code is specified by this matcher. This matcher will be used to assert the status code when + * the expected set of indices is empty. + */ + protected final RestMatchers.HttpResponseMatcher statusCodeWhenEmpty; + + /** + * This is true if we also expect the .opendistro_security index. In case we gain further + * system indices that are present by default on an int test cluster, this can be expanded to cover also these. + */ + protected final boolean containsOpenSearchSecurityIndex; + + AbstractIndexMatcher(Map expectedIndices, boolean containsOpenSearchSecurityIndex) { + this.expectedIndices = expectedIndices; + this.jsonPath = null; + this.statusCodeWhenEmpty = RestMatchers.isOk(); + this.containsOpenSearchSecurityIndex = containsOpenSearchSecurityIndex; + } + + AbstractIndexMatcher( + Map expectedIndices, + boolean containsOpenSearchSecurityIndex, + String jsonPath, + RestMatchers.HttpResponseMatcher statusCodeWhenEmpty + ) { + this.expectedIndices = expectedIndices; + this.jsonPath = jsonPath; + this.statusCodeWhenEmpty = statusCodeWhenEmpty; + this.containsOpenSearchSecurityIndex = containsOpenSearchSecurityIndex; + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + TestRestClient.HttpResponse response = null; + + if (item instanceof TestRestClient.HttpResponse) { + response = (TestRestClient.HttpResponse) item; + + if (expectedIndices.isEmpty()) { + if (response.getStatusCode() != this.statusCodeWhenEmpty.statusCode()) { + mismatchDescription.appendText("Status was: ") + .appendValue(response.getStatusCode() + " " + response.getStatusReason()) + .appendText("\n\n") + .appendText(formatResponse(response)); + return false; + } + + if (response.getStatusCode() != 200) { + return true; + } + } + + try { + if (response.getBody().startsWith(START_ARRAY.asString())) { + item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), List.class); + } else { + item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), Map.class); + } + } catch (JsonProcessingException e) { + mismatchDescription.appendText("Unable to parse body: ").appendValue(e.getMessage()); + return false; + } + } + + if (jsonPath != null) { + Configuration config = Configuration.builder() + .jsonProvider(new JacksonJsonProvider()) + .mappingProvider(new JacksonMappingProvider()) + .evaluationListener() + .options(Option.SUPPRESS_EXCEPTIONS) + .build(); + + item = JsonPath.using(config).parse(item).read(jsonPath); + + if (item == null) { + mismatchDescription.appendText("Unable to find JSON Path: ") + .appendValue(jsonPath) + .appendText("\n\n") + .appendText(formatResponse(response)); + return false; + } + } + + if (!(item instanceof Collection)) { + item = Collections.singleton(item); + } + + return matchesImpl((Collection) item, mismatchDescription, response); + } + + /** + * This is called by the main matches() method after the indices have been extracted + * from the HTTP response body. The found indices will be passed as the actualItems parameter. + * + * @param actualItems The found indices. This is expected to be strings. + * @param mismatchDescription In case the matcher finds a mismatch, the description should be appended to this object. + * @param response The REST response we are asserting against. Optional. + * @return true if the assertion was successful, false it it failed. + */ + protected abstract boolean matchesImpl( + Collection actualItems, + Description mismatchDescription, + TestRestClient.HttpResponse response + ); + + @Override + public boolean isEmpty() { + return expectedIndices.isEmpty(); + } + + @Override + public int size() { + if (!containsOpenSearchSecurityIndex) { + return expectedIndices.size(); + } else { + throw new RuntimeException("Size cannot be exactly specified because containsOpenSearchIndices is true"); + } + } + + @Override + public boolean containsOpenSearchSecurityIndex() { + return containsOpenSearchSecurityIndex; + } + + /** + * Calculates the intersection of the two given Map objects. + */ + protected Map testIndicesIntersection( + Map map1, + Map map2 + ) { + Map result = new HashMap<>(); + + for (Map.Entry entry : map1.entrySet()) { + String key = entry.getKey(); + TestIndexOrAliasOrDatastream index1 = entry.getValue(); + TestIndexOrAliasOrDatastream index2 = map2.get(key); + + if (index2 == null) { + continue; + } + + result.put(key, index1.intersection(index2)); + } + + return Collections.unmodifiableMap(result); + } + + protected ImmutableSet getExpectedIndices() { + return ImmutableSet.copyOf(expectedIndices.keySet()); + } + + /** + * Returns a formatted version of the response. This can be used in the mismatch description. + */ + protected static String formatResponse(TestRestClient.HttpResponse response) { + if (response == null) { + return ""; + } + + String start = response.getStatusCode() + " " + response.getStatusReason() + "\n"; + + if (response.isJsonContentType()) { + return start + response.bodyAsJsonNode().toPrettyString(); + } else { + return start + response.getBody(); + } + } + } + + /** + * This asserts that the item we assert on contains a set of indices that exactly corresponds to the expected + * indices (i.e., not fewer and not more indices). This is usually used to match against REST responses. + */ static class ContainsExactlyMatcher extends AbstractIndexMatcher implements OnResponseIndexMatcher { private static final Pattern DS_BACKING_INDEX_PATTERN = Pattern.compile("\\.ds-(.+)-[0-9]+"); @@ -197,7 +407,7 @@ static class ContainsExactlyMatcher extends AbstractIndexMatcher implements OnRe @Override public void describeTo(Description description) { - if (indexNameMap.isEmpty()) { + if (expectedIndices.isEmpty()) { if (this.statusCodeWhenEmpty.statusCode() == 200) { description.appendText("a 200 OK response with an empty result set"); } else { @@ -206,19 +416,19 @@ public void describeTo(Description description) { } } else { description.appendText( - "a 200 OK response with exactly the indices " + indexNameMap.keySet().stream().collect(Collectors.joining(", ")) + "a 200 OK response with exactly the indices " + expectedIndices.keySet().stream().collect(Collectors.joining(", ")) ); } } @Override - protected boolean matchesImpl(Collection collection, Description mismatchDescription, TestRestClient.HttpResponse response) { + protected boolean matchesImpl(Collection actualItems, Description mismatchDescription, TestRestClient.HttpResponse response) { // Flatten the collection - collection = collection.stream() + actualItems = actualItems.stream() .flatMap(e -> e instanceof Collection ? ((Collection) e).stream() : Stream.of(e)) .collect(Collectors.toSet()); - return matchesByIndices(collection, mismatchDescription, response); + return matchesByIndices(actualItems, mismatchDescription, response); } protected boolean matchesByIndices( @@ -233,7 +443,7 @@ protected boolean matchesByIndices( for (Object object : collection) { String index = object.toString(); - if (containsOpenSearchIndices && (index.startsWith(".opendistro"))) { + if (containsOpenSearchSecurityIndex && (index.startsWith(".opendistro"))) { seenOpenSearchIndicesBuilder.add(index); } else if (index.startsWith(".ds-")) { // We do a special treatment for data stream backing indices. We convert these to the normal data streams if expected @@ -255,7 +465,7 @@ protected boolean matchesByIndices( ImmutableSet unexpectedIndices = Sets.difference(seenIndices, expectedIndices).immutableCopy(); ImmutableSet missingIndices = Sets.difference(expectedIndices, seenIndices).immutableCopy(); - if (containsOpenSearchIndices && seenOpenSearchIndicesBuilder.build().size() == 0) { + if (containsOpenSearchSecurityIndex && seenOpenSearchIndicesBuilder.build().size() == 0) { missingIndices = ImmutableSet.builderWithExpectedSize(missingIndices.size() + 1) .addAll(missingIndices) .add(".opensearch indices") @@ -288,22 +498,22 @@ protected boolean matchesByIndices( public OnResponseIndexMatcher reducedBy(IndexMatcher other) { if (other instanceof LimitedToMatcher) { return new ContainsExactlyMatcher( - testIndicesIntersection(this.indexNameMap, ((LimitedToMatcher) other).indexNameMap), // - this.containsOpenSearchIndices && other.containsOpenSearchIndices(), // + testIndicesIntersection(this.expectedIndices, ((LimitedToMatcher) other).expectedIndices), // + this.containsOpenSearchSecurityIndex && other.containsOpenSearchSecurityIndex(), // this.jsonPath, this.statusCodeWhenEmpty ); } else if (other instanceof ContainsExactlyMatcher) { return new ContainsExactlyMatcher( - testIndicesIntersection(this.indexNameMap, ((ContainsExactlyMatcher) other).indexNameMap), // - this.containsOpenSearchIndices && other.containsOpenSearchIndices(), // + testIndicesIntersection(this.expectedIndices, ((ContainsExactlyMatcher) other).expectedIndices), // + this.containsOpenSearchSecurityIndex && other.containsOpenSearchSecurityIndex(), // this.jsonPath, this.statusCodeWhenEmpty ); } else if (other instanceof UnlimitedMatcher) { return new ContainsExactlyMatcher( - this.indexNameMap, // - this.containsOpenSearchIndices && other.containsOpenSearchIndices(), // + this.expectedIndices, // + this.containsOpenSearchSecurityIndex && other.containsOpenSearchSecurityIndex(), // this.jsonPath, this.statusCodeWhenEmpty ); @@ -314,17 +524,17 @@ public OnResponseIndexMatcher reducedBy(IndexMatcher other) { @Override public OnResponseIndexMatcher at(String jsonPath) { - return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchIndices, jsonPath, statusCodeWhenEmpty); + return new ContainsExactlyMatcher(expectedIndices, containsOpenSearchSecurityIndex, jsonPath, statusCodeWhenEmpty); } @Override public OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode) { - return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchIndices, jsonPath, statusCode); + return new ContainsExactlyMatcher(expectedIndices, containsOpenSearchSecurityIndex, jsonPath, statusCode); } @Override public boolean covers(TestIndexOrAliasOrDatastream testIndex) { - return indexNameMap.containsKey(testIndex.name()); + return expectedIndices.containsKey(testIndex.name()); } @Override @@ -333,8 +543,8 @@ public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatche return this; } - HashMap unmatched = new HashMap<>(this.indexNameMap); - unmatched.keySet().removeAll(((AbstractIndexMatcher) other).indexNameMap.keySet()); + HashMap unmatched = new HashMap<>(this.expectedIndices); + unmatched.keySet().removeAll(((AbstractIndexMatcher) other).expectedIndices.keySet()); if (!unmatched.isEmpty()) { return new StatusCodeMatcher(statusCode); @@ -344,6 +554,11 @@ public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatche } } + /** + * Just asserts on the status code of a response. This is usually only used for failure status codes which + * are expected when the expected set of indices is empty. In this case, we do not apply any JSON path + * extractions, as we expect the response body to be just an error message. + */ static class StatusCodeMatcher extends DiagnosingMatcher implements OnResponseIndexMatcher { private RestMatchers.HttpResponseMatcher expectedStatusCode; @@ -367,7 +582,7 @@ public boolean isEmpty() { } @Override - public boolean containsOpenSearchIndices() { + public boolean containsOpenSearchSecurityIndex() { return true; } @@ -402,6 +617,11 @@ public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatche } } + /** + * This asserts that the item we assert on contains not more than the expected indices. + * Usually, this is only associated with TestUser objects and used to reduce ContainsExactly matchers + * to even more limited ContainsExactly matchers. + */ static class LimitedToMatcher extends AbstractIndexMatcher implements OnUserIndexMatcher { LimitedToMatcher(Map indexNameMap) { @@ -410,7 +630,7 @@ static class LimitedToMatcher extends AbstractIndexMatcher implements OnUserInde @Override public void describeTo(Description description) { - if (indexNameMap.isEmpty()) { + if (expectedIndices.isEmpty()) { if (this.statusCodeWhenEmpty.statusCode() == 200) { description.appendText("a 200 OK response with an empty result set"); } else { @@ -418,19 +638,19 @@ public void describeTo(Description description) { } } else { description.appendText( - "a 200 OK response no indices other than " + indexNameMap.keySet().stream().collect(Collectors.joining(", ")) + "a 200 OK response no indices other than " + expectedIndices.keySet().stream().collect(Collectors.joining(", ")) ); } } @Override - protected boolean matchesImpl(Collection collection, Description mismatchDescription, TestRestClient.HttpResponse response) { - return matchesByIndices(collection, mismatchDescription, response); + protected boolean matchesImpl(Collection actualItems, Description mismatchDescription, TestRestClient.HttpResponse response) { + return matchesByIndices(actualItems, mismatchDescription, response); } @Override public boolean covers(TestIndexOrAliasOrDatastream testIndex) { - return indexNameMap.containsKey(testIndex.name()); + return expectedIndices.containsKey(testIndex.name()); } protected boolean matchesByIndices( @@ -460,6 +680,10 @@ protected boolean matchesByIndices( } } + /** + * This does no assertion on the expected indices. Usually, this is only associated with TestUser objects and used + * to signal that ContainsExactly matchers do not need to be reduced. + */ static class UnlimitedMatcher extends DiagnosingMatcher implements OnUserIndexMatcher { private final boolean containsOpenSearchIndices; @@ -479,9 +703,7 @@ public void describeTo(Description description) { @Override protected boolean matches(Object item, Description mismatchDescription) { - if (item instanceof TestRestClient.HttpResponse) { - TestRestClient.HttpResponse response = (TestRestClient.HttpResponse) item; - + if (item instanceof TestRestClient.HttpResponse response) { if (response.getStatusCode() != 200) { mismatchDescription.appendText("Expected status code 200 but status was: ") .appendValue(response.getStatusCode() + " " + response.getStatusReason()); @@ -498,7 +720,7 @@ public boolean isEmpty() { } @Override - public boolean containsOpenSearchIndices() { + public boolean containsOpenSearchSecurityIndex() { return containsOpenSearchIndices; } @@ -512,154 +734,4 @@ public boolean covers(TestIndexOrAliasOrDatastream testIndex) { return true; } } - - static abstract class AbstractIndexMatcher extends DiagnosingMatcher implements IndexMatcher { - protected final Map indexNameMap; - protected final String jsonPath; - protected final RestMatchers.HttpResponseMatcher statusCodeWhenEmpty; - protected final boolean containsOpenSearchIndices; - - AbstractIndexMatcher(Map indexNameMap, boolean containsOpenSearchIndices) { - this.indexNameMap = indexNameMap; - this.jsonPath = null; - this.statusCodeWhenEmpty = RestMatchers.isOk(); - this.containsOpenSearchIndices = containsOpenSearchIndices; - } - - AbstractIndexMatcher( - Map indexNameMap, - boolean containsOpenSearchIndices, - String jsonPath, - RestMatchers.HttpResponseMatcher statusCodeWhenEmpty - ) { - this.indexNameMap = indexNameMap; - this.jsonPath = jsonPath; - this.statusCodeWhenEmpty = statusCodeWhenEmpty; - this.containsOpenSearchIndices = containsOpenSearchIndices; - } - - @Override - protected boolean matches(Object item, Description mismatchDescription) { - TestRestClient.HttpResponse response = null; - - if (item instanceof TestRestClient.HttpResponse) { - response = (TestRestClient.HttpResponse) item; - - if (indexNameMap.isEmpty()) { - if (response.getStatusCode() != this.statusCodeWhenEmpty.statusCode()) { - mismatchDescription.appendText("Status was: ") - .appendValue(response.getStatusCode() + " " + response.getStatusReason()) - .appendText("\n\n") - .appendText(formatResponse(response)); - return false; - } - - if (response.getStatusCode() != 200) { - return true; - } - } - - try { - if (response.getBody().startsWith(START_ARRAY.asString())) { - item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), List.class); - } else { - item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), Map.class); - } - } catch (JsonProcessingException e) { - mismatchDescription.appendText("Unable to parse body: ").appendValue(e.getMessage()); - return false; - } - } - - if (jsonPath != null) { - Configuration config = Configuration.builder() - .jsonProvider(new JacksonJsonProvider()) - .mappingProvider(new JacksonMappingProvider()) - .evaluationListener() - .options(Option.SUPPRESS_EXCEPTIONS) - .build(); - - item = JsonPath.using(config).parse(item).read(jsonPath); - - if (item == null) { - mismatchDescription.appendText("Unable to find JSON Path: ") - .appendValue(jsonPath) - .appendText("\n\n") - .appendText(formatResponse(response)); - return false; - } - } - - if (!(item instanceof Collection)) { - item = Collections.singleton(item); - } - - return matchesImpl((Collection) item, mismatchDescription, response); - } - - protected abstract boolean matchesImpl( - Collection collection, - Description mismatchDescription, - TestRestClient.HttpResponse response - ); - - @Override - public boolean isEmpty() { - return indexNameMap.isEmpty(); - } - - @Override - public int size() { - if (!containsOpenSearchIndices) { - return indexNameMap.size(); - } else { - throw new RuntimeException("Size cannot be exactly specified because containsOpenSearchIndices is true"); - } - } - - @Override - public boolean containsOpenSearchIndices() { - return containsOpenSearchIndices; - } - - protected Map testIndicesIntersection( - Map map1, - Map map2 - ) { - Map result = new HashMap<>(); - - for (Map.Entry entry : map1.entrySet()) { - String key = entry.getKey(); - TestIndexOrAliasOrDatastream index1 = entry.getValue(); - TestIndexOrAliasOrDatastream index2 = map2.get(key); - - if (index2 == null) { - continue; - } - - result.put(key, index1.intersection(index2)); - } - - return Collections.unmodifiableMap(result); - } - - protected ImmutableSet getExpectedIndices() { - return ImmutableSet.copyOf(indexNameMap.keySet()); - } - - } - - private static String formatResponse(TestRestClient.HttpResponse response) { - if (response == null) { - return ""; - } - - String start = response.getStatusCode() + " " + response.getStatusReason() + "\n"; - - if (response.isJsonContentType()) { - return start + response.bodyAsJsonNode().toPrettyString(); - } else { - return start + response.getBody(); - } - } } diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties index 18ef360ba2..1ff778cb1c 100644 --- a/src/integrationTest/resources/log4j2-test.properties +++ b/src/integrationTest/resources/log4j2-test.properties @@ -48,6 +48,9 @@ logger.ldap.name=org.opensearch.security.auth.ldap.backend logger.ldap.level=TRACE logger.ldap.appenderRef.capturing.ref = logCapturingAppender +logger.privileges.name=org.opensearch.security.privileges +logger.privileges.level=DEBUG + # Logger required by test org.opensearch.security.TlsHostnameVerificationTests logger.securenetty4transport.name = org.opensearch.transport.netty4.ssl.SecureNetty4Transport logger.securenetty4transport.level = error From 571c6e73116ecf0d2e38c8c8d66f7c857bafb607 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 1 Oct 2025 09:38:57 +0200 Subject: [PATCH 11/22] Rollover tests Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 122 +++++++++--------- .../IndexAuthorizationReadWriteIntTests.java | 89 ++++++++----- .../opensearch/test/framework/TestAlias.java | 1 - .../opensearch/test/framework/TestData.java | 10 +- 4 files changed, 128 insertions(+), 94 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index b2f679b5f1..e3d755770a 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -14,7 +14,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; @@ -630,17 +629,13 @@ public void search_staticIndices_systemIndex_alias() throws Exception { if (user == SUPER_UNLIMITED_USER) { assertThat(httpResponse, isOk()); - assertThat( - httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index")); + assertThat(httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index")); } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { - if (user == UNLIMITED_USER || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + if (user == UNLIMITED_USER || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, // but withholds documents on the DLS level assertThat(httpResponse, isOk()); - assertThat( - httpResponse, - containsExactly().at("hits.hits[*]._index")); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); } else { assertThat(httpResponse, isForbidden()); } @@ -652,15 +647,16 @@ public void search_staticIndices_systemIndex_alias() throws Exception { assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); } else { assertThat( - httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isForbidden()) + ); } } else { if (user.indexMatcher("read").covers(alias_with_system_index)) { assertThat(httpResponse, isOk()); - assertThat( - httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index")); + assertThat(httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index")); } else { assertThat(httpResponse, isForbidden()); } @@ -852,19 +848,20 @@ public void search_alias_pattern_negation() throws Exception { /* if (user != LIMITED_USER_NONE) { if (clusterConfig.systemIndexPrivilegeEnabled) { - // If the system index privilege is enabled, we might also see the system_index_plugin index (being included via the alias) + // If the system index privilege is enabled, we might also see the system_index_plugin index (being included via the + // alias) assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } else { @@ -881,26 +878,29 @@ public void search_alias_pattern_includeHidden() throws Exception { if (user == SUPER_UNLIMITED_USER) { assertThat(httpResponse, isOk()); assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index")); + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") + ); } else if (user != LIMITED_USER_NONE) { if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { // For all users without the system index permission, SystemIndexAccessEvaluator shuts the door - // For the user with the system index permission, that happens as well, as SystemIndexAccessEvaluator expects the permission for all requested indices, even if they are not system indices - assertThat(httpResponse, isForbidden()); + // For the user with the system index permission, that happens as well, as SystemIndexAccessEvaluator expects the + // permission for all requested indices, even if they are not system indices + assertThat(httpResponse, isForbidden()); } else { assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(isOk())); + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isOk()) + ); } } else { assertThat(httpResponse, isForbidden()); @@ -908,7 +908,6 @@ public void search_alias_pattern_includeHidden() throws Exception { } } - @Test public void search_aliasAndIndex_ignoreUnavailable() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { @@ -1120,33 +1119,32 @@ public void search_pit_wrongIndex() throws Exception { public void search_template_staticIndices() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { String params = """ - { - "department": [%s] - }""".formatted(TestData.DEPARTMENTS.stream().map(s -> '"' + s + '"').collect(joining(","))); + { + "department": [%s] + }""".formatted(TestData.DEPARTMENTS.stream().map(s -> '"' + s + '"').collect(joining(","))); String query = """ - { - "query": { - "terms": { - "attr_text_1": [ - "{{#department}}", - "{{.}}", - "{{/department}}" - ] - } - } + { + "query": { + "terms": { + "attr_text_1": [ + "{{#department}}", + "{{.}}", + "{{/department}}" + ] } - """; + } + } + """; - TestRestClient.HttpResponse httpResponse = restClient.getWithJsonBody("index_a1/_search/template?size=1000", - """ - { - "params": %s, - "source": "%s" - }""".formatted(params, escapeJson(query))); + TestRestClient.HttpResponse httpResponse = restClient.getWithJsonBody("index_a1/_search/template?size=1000", """ + { + "params": %s, + "source": "%s" + }""".formatted(params, escapeJson(query))); assertThat( - httpResponse, - containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + httpResponse, + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) ); } } @@ -1336,14 +1334,14 @@ public void cat_aliases_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases?format=json"); if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { - assertThat( - httpResponse, - containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias")); + assertThat(httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias")); } else { if (!user.indexMatcher("get_alias").isEmpty()) { assertThat( - httpResponse, - containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isOk()) + httpResponse, + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias") + .reducedBy(user.indexMatcher("get_alias")) + .whenEmpty(isOk()) ); } else { assertThat(httpResponse, isForbidden()); diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index 7c81b6c5bc..5dae750b31 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -185,7 +185,7 @@ public class IndexAuthorizationReadWriteIntTests { .on("index_bw*")// .indexPermissions("manage") .on("index_bw*")// - .indexPermissions("manage_aliases") + .indexPermissions("crud", "manage", "manage_aliases") .on("alias_bwx*") )// .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// @@ -195,6 +195,31 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// .indexMatcher("get_alias", limitedTo(alias_bwx)); + /** + * This user differs from LIMITED_USER_B_MANAGE_INDEX_ALIAS the way that it does not give any direct + * write privileges to index_bw*; rather, it gives write privileges to alias_bxw. Any index which happens + * to be member of that alias then gains these write privileges. + */ + static TestSecurityConfig.User LIMITED_USER_B_READ_ONLY_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( + "limited_user_B_index_read_only_manage_index_alias" + )// + .description("index_b* r/o, alias_bwx* r/w with manage privs")// + .roles( + // + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("crud", "manage", "manage_aliases") + .on("alias_bwx*") + )// + .indexMatcher("read", limitedTo(index_br1, index_br2))// + .indexMatcher("write", limitedToNone())// + .indexMatcher("create_index", limitedToNone())// + .indexMatcher("manage_index", limitedTo(alias_bwx))// + .indexMatcher("manage_alias", limitedTo(alias_bwx))// + .indexMatcher("get_alias", limitedTo(alias_bwx)); + static TestSecurityConfig.User LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( "limited_user_B_hidden_manage_index_alias" )// @@ -209,10 +234,25 @@ public class IndexAuthorizationReadWriteIntTests { .on("index_bw*", "index_hidden*")// .indexPermissions("manage") .on("index_bw*", "index_hidden*")// - .indexPermissions("manage_aliases") + .indexPermissions("crud", "manage", "manage_aliases") .on("alias_bwx*") )// - .indexMatcher("read", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .indexMatcher( + "read", + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_hidden + ) + )// .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// @@ -567,6 +607,7 @@ public class IndexAuthorizationReadWriteIntTests { LIMITED_USER_B_CREATE_INDEX, LIMITED_USER_B_MANAGE_INDEX, LIMITED_USER_B_MANAGE_INDEX_ALIAS, + LIMITED_USER_B_READ_ONLY_MANAGE_INDEX_ALIAS, LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS, LIMITED_USER_B_SYSTEM_INDEX_MANAGE, LIMITED_USER_C, @@ -852,10 +893,10 @@ public void deleteIndex() throws Exception { createInitialTestObjects(index_bwx1); HttpResponse httpResponse = restClient.delete("index_bwx1"); - if (user.indexMatcher("manage_index").isEmpty()) { - assertThat(httpResponse, isForbidden()); - } else { + if (user.indexMatcher("manage_index").covers(index_bwx1)) { assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); } } finally { delete(index_bwx1); @@ -911,7 +952,7 @@ public void deleteAlias_staticIndex() throws Exception { HttpResponse httpResponse = restClient.delete("index_bw1/_aliases/alias_bwx"); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bw1)) { + if (user.indexMatcher("manage_alias").covers(index_bw1) || user.indexMatcher("manage_alias").covers(alias_bwx)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -1000,7 +1041,7 @@ public void aliases_deleteAlias_staticIndex() throws Exception { }"""); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bw1)) { + if (user.indexMatcher("manage_alias").covers(index_bw1) || user.indexMatcher("manage_alias").covers(alias_bwx)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -1158,44 +1199,32 @@ public void closeIndex_openIndex() throws Exception { @Test public void rollover_explicitTargetIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - createInitialTestObjects(alias_bwx.on(index_bw1, index_bw2)); + createInitialTestObjects(alias_bwx.on(index_bw1).writeIndex(index_bw1)); - HttpResponse httpResponse = restClient.postJson("_aliases", """ + HttpResponse httpResponse = restClient.postJson("alias_bwx/_rollover/index_bwx1", """ { - "actions": [ - { "remove": { "index": "*", "alias": "alias_bwx" } } - ] + "conditions": { + "max_age": "0s" + } }"""); + System.out.println(httpResponse.getBody()); + if (clusterConfig.legacyPrivilegeEvaluation) { - // This is only allowed if we have privileges for all indices, even if not all indices are member of alias_bwx - if (user.indexMatcher("manage_alias") - .coversAll( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw1, - index_cr1, - index_cw1 - )) { + if (user.indexMatcher("manage_alias").covers(index_bw1) && user.indexMatcher("manage_index").covers(index_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); } } else { - if (user.indexMatcher("manage_alias").coversAll(alias_bwx)) { + if (user.indexMatcher("manage_alias").covers(alias_bwx) && user.indexMatcher("manage_index").covers(index_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); } } - } finally { - delete(alias_bwx); + delete(alias_bwx, index_bwx1); } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java b/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java index a763a365e2..aff626f444 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableSet; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.opensearch.common.settings.Settings; import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; import org.opensearch.transport.client.Client; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestData.java b/src/integrationTest/java/org/opensearch/test/framework/TestData.java index 7c28d7912e..0e949c2734 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestData.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestData.java @@ -93,7 +93,15 @@ public static TestData.Builder documentCount(int documentCount) { "attr_object.obj_attr_object.obj_obj_attr_text.keyword" ); - public static final ImmutableList DEPARTMENTS = ImmutableList.of("dept_a_1", "dept_a_2", "dept_a_3", "dept_b_1", "dept_b_2", "dept_c", "dept_d"); + public static final ImmutableList DEPARTMENTS = ImmutableList.of( + "dept_a_1", + "dept_a_2", + "dept_a_3", + "dept_b_1", + "dept_b_2", + "dept_c", + "dept_d" + ); private static final Cache cache; From 5cb503605d067c147c9901e3b03826391141bf5e Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Thu, 2 Oct 2025 13:10:42 +0200 Subject: [PATCH 12/22] PIT tests Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 216 ++++++++++--- .../IndexAuthorizationReadWriteIntTests.java | 288 ++++++------------ .../framework/cluster/TestRestClient.java | 12 + .../matcher/IndexApiResponseMatchers.java | 37 ++- 4 files changed, 322 insertions(+), 231 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index e3d755770a..7519579a3e 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -11,9 +11,12 @@ package org.opensearch.security.privileges.int_tests; +import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.stream.Stream; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; @@ -41,6 +44,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.IndexMatcher; import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; @@ -67,7 +71,10 @@ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class IndexAuthorizationReadOnlyIntTests { - // TODO pit_segments + // ------------------------------------------------------------------------------------------------------- + // Test indices used by this test suite. Indices are usually initially created; the only exception is + // index_ax, which is referred to in tests, but which does not exist on purpose. + // ------------------------------------------------------------------------------------------------------- static final TestIndex index_a1 = TestIndex.name("index_a1").documentCount(100).seed(1).build(); static final TestIndex index_a2 = TestIndex.name("index_a2").documentCount(110).seed(2).build(); @@ -128,6 +135,30 @@ public class IndexAuthorizationReadOnlyIntTests { openSearchSecurityConfigIndex() ); + static final List ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot + ); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from index_a* + */ static final TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("index_a*")// .roles( @@ -139,6 +170,9 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax))// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple user that can read from index_b* + */ static final TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("index_b*")// .roles( @@ -150,6 +184,9 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedTo(index_b1, index_b2, index_b3))// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple user that can read only from index_b1 + */ static final TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// .description("index_b1")// .roles( @@ -161,6 +198,9 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedTo(index_b1))// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple user that can read from index_c* + */ static final TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// .description("index_c*")// .roles( @@ -172,6 +212,10 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedTo(index_c1, alias_c1))// .indexMatcher("get_alias", limitedToNone()); + /** + * A user that has read privileges for alias_ab1*; these privileges are inherited to the member indices. + * The user has no directly defined privileges on indices. + */ static final TestSecurityConfig.User LIMITED_USER_ALIAS_AB1 = new TestSecurityConfig.User("limited_user_alias_AB1")// .description("alias_ab1")// .roles( @@ -183,6 +227,10 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// .indexMatcher("get_alias", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1)); + /** + * A user that has read privileges for alias_c1; these privileges are inherited to the member indices. + * The user has no directly defined privileges on indices. + */ static final TestSecurityConfig.User LIMITED_USER_ALIAS_C1 = new TestSecurityConfig.User("limited_user_alias_C1")// .description("alias_c1")// .roles( @@ -193,7 +241,9 @@ public class IndexAuthorizationReadOnlyIntTests { )// .indexMatcher("read", limitedTo(index_c1, alias_c1))// .indexMatcher("get_alias", limitedTo(index_c1, alias_c1)); - + /** + * Same as LIMITED_USER_A with the addition of read privileges for index_hidden* and .index_hidden* + */ static final TestSecurityConfig.User LIMITED_USER_A_HIDDEN = new TestSecurityConfig.User("limited_user_A_hidden")// .description("index_a*, index_hidden*")// .roles( @@ -205,6 +255,10 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// .indexMatcher("get_alias", limitedToNone()); + /** + * Same as LIMITED_USER_C with the addition of read privileges for ".system_index_plugin"; they also have the + * explicit privilege "system:admin/system_index" that allows them accessing this index. + */ static final TestSecurityConfig.User LIMITED_USER_C_WITH_SYSTEM_INDICES = new TestSecurityConfig.User( "limited_user_C_with_system_indices" )// @@ -243,6 +297,9 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * This user has no index read privileges at all. + */ static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// .description("no index privileges")// .roles( @@ -252,49 +309,21 @@ public class IndexAuthorizationReadOnlyIntTests { .indexMatcher("read", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// .roles( new TestSecurityConfig.Role("r1")// - .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .clusterPermissions("*") .indexPermissions("*") .on("*")// )// - .indexMatcher( - "read", - limitedTo( - index_a1, - index_a2, - index_a3, - index_b1, - index_b2, - index_b3, - index_c1, - alias_ab1, - alias_c1, - index_hidden, - index_hidden_dot, - index_ax - ) - )// - .indexMatcher( - "get_alias", - limitedTo( - index_a1, - index_a2, - index_a3, - index_b1, - index_b2, - index_b3, - index_c1, - alias_ab1, - alias_c1, - index_hidden, - index_hidden_dot, - index_ax - ) - ); + .indexMatcher("read", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax))// + .indexMatcher("get_alias", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax)); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -1766,6 +1795,102 @@ public void field_caps_indexPattern_minus() throws Exception { } } + @Test + public void pit_list_all() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search/point_in_time/_all"); + + if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { + // Once again, the system index privilege code makes it impossible to use this action without super admin privileges + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else { + // The behavior in legacy privilege evaluation and new privilege evaluation actually differs, even though we do not observe + // here a difference: + // - Legacy: the user needs to have the privilege for all indices. If it is only granted for a subset of indices, this will + // be forbidden. + // - New: this is now a cluster privilege, the users below are the users with full cluster privileges + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + deletePit(indexA1pitId); + } + } + + @Test + public void pit_delete() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.delete("_search/point_in_time", json("pit_id", List.of(indexA1pitId))); + + if (user.indexMatcher("read").covers(index_a1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deletePit(indexA1pitId); + } + } + + @Test + public void pit_catSegments() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/pit_segments", json("pit_id", List.of(indexA1pitId))); + + if (user.indexMatcher("read").covers(index_a1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deletePit(indexA1pitId); + } + } + + @Test + public void pit_catSegments_all() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/pit_segments/_all"); + + if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { + // Once again, the system index privilege code makes it impossible to use this action without super admin privileges + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else { + // The behavior in legacy privilege evaluation and new privilege evaluation actually differs, even though we do not observe + // here a difference: + // - Legacy: the user needs to have the privilege for all indices. If it is only granted for a subset of indices, this will + // be forbidden. + // - New: this is now a separate cluster privilege, the users below are the users with full cluster privileges + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + deletePit(indexA1pitId); + } + } + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") public static Collection params() { List result = new ArrayList<>(); @@ -1794,4 +1919,21 @@ public Collection getSystemIndexDescriptors(Settings sett ); } } + + private String createPit(TestIndex... indices) throws IOException { + try (TestRestClient client = cluster.getAdminCertRestClient()) { + TestRestClient.HttpResponse response = client.post( + Stream.of(indices).map(TestIndex::name).collect(joining(",")) + "/_search/point_in_time?keep_alive=1m" + ); + assertThat(response, isOk()); + return response.getTextFromJsonBody("/pit_id"); + } + } + + private void deletePit(String... pitIds) { + try (TestRestClient client = cluster.getAdminCertRestClient()) { + TestRestClient.HttpResponse response = client.delete("_search/point_in_time", json("pit_id", Arrays.asList(pitIds))); + assertThat(response, isOk()); + } + } } diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index 5dae750b31..d7a6760f4f 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -69,6 +69,13 @@ @NotThreadSafe public class IndexAuthorizationReadWriteIntTests { + // ------------------------------------------------------------------------------------------------------- + // Test indices used by this test suite. We use the following naming scheme: + // - index_*r*: This test will not write to this index + // - index_*w*: This test can write to this index; the test won't delete and recreate it + // - index_*wx*: The index is not initially created; the test can create it on demand and delete it again + // ------------------------------------------------------------------------------------------------------- + static final TestIndex index_ar1 = TestIndex.name("index_ar1").documentCount(10).build(); static final TestIndex index_ar2 = TestIndex.name("index_ar2").documentCount(10).build(); static final TestIndex index_aw1 = TestIndex.name("index_aw1").documentCount(10).build(); @@ -97,10 +104,50 @@ public class IndexAuthorizationReadWriteIntTests { static final TestAlias alias_bwx = new TestAlias("alias_bwx"); // not initially created + static final List ALL_NON_HIDDEN_INDICES = List.of( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_cr1, + index_cw1 + ); + + static final List ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES = List.of( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_ab1w_nowriteindex, + index_hidden + ); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from index_a* and write to index_aw*; the user as no privileges to create or manage indices + */ static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("index_a*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -115,10 +162,12 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple user that can read from index_b* and write to index_bw*; the user as no privileges to create or manage indices + */ static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("index_b*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -133,10 +182,12 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple user that can read from index_b* and write to index_bw*; additionally, they can create index_bw* indices + */ static TestSecurityConfig.User LIMITED_USER_B_CREATE_INDEX = new TestSecurityConfig.User("limited_user_B_create_index")// .description("index_b* with create index privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -153,10 +204,12 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple user that can read from index_b* and write to index_bw*; additionally, they can create and manage index_bw* indices + */ static TestSecurityConfig.User LIMITED_USER_B_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_B_manage_index")// .description("index_b* with manage privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -173,10 +226,14 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// .indexMatcher("get_alias", limitedTo()); + /** + * A user that can read from index_b* and write to index_bw*; they can create and manage index_bw* indices and manage alias_bwx* aliases. + * For users with such alias permissions, keep in mind that alias permissions are inherited by the member indices. + * Thus, indices can gain or lose privileges when they are added/removed from the alias. + */ static TestSecurityConfig.User LIMITED_USER_B_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User("limited_user_B_manage_index_alias")// .description("index_b*, alias_bwx* with manage privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -205,7 +262,6 @@ public class IndexAuthorizationReadWriteIntTests { )// .description("index_b* r/o, alias_bwx* r/w with manage privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -220,12 +276,14 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedTo(alias_bwx))// .indexMatcher("get_alias", limitedTo(alias_bwx)); + /** + * Same as LIMITED_USER_B_MANAGE_INDEX_ALIAS with the addition of read/write/manage privileges on index_hidden* + */ static TestSecurityConfig.User LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( "limited_user_B_hidden_manage_index_alias" )// .description("index_b*, index_hidden*, alias_bwx* with manage privs, index_a* read only")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") @@ -259,10 +317,13 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// .indexMatcher("get_alias", limitedTo(alias_bwx)); + /** + * Same as LIMITED_USER_B with the addition of read/write/manage privileges for ".system_index_plugin", ".system_index_plugin_*" + * including the explicit "system:admin/system_index" privilege. + */ static TestSecurityConfig.User LIMITED_USER_B_SYSTEM_INDEX_MANAGE = new TestSecurityConfig.User("limited_user_B_system_index_manage")// .description("index_b*, .system_index_plugin with manage privs")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/refresh*", "system:admin/system_index") @@ -303,28 +364,13 @@ public class IndexAuthorizationReadWriteIntTests { )// .indexMatcher("get_alias", limitedToNone()); - static TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// - .description("index_c*")// - .roles( - // - new Role("r1")// - .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh") - .on("index_c*")// - .indexPermissions("write") - .on("index_cw*") - )// - .indexMatcher("read", limitedTo(index_cr1, index_cw1))// - .indexMatcher("write", limitedTo(index_cw1))// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); - + /** + * A simple test user that has read privileges on alias_ab1r and write privileges on alias_ab1w*. The user + * has no direct privileges on indices; all privileges are gained via the aliases. + */ static TestSecurityConfig.User LIMITED_USER_AB1_ALIAS = new TestSecurityConfig.User("limited_user_alias_AB1")// .description("alias_ab1")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get") @@ -342,10 +388,13 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1, alias_ab1r, alias_ab1w)); + /** + * A simple test user that has read/only privileges on alias_ab1r and alias_ab1w*. However, they have write + * privileges for the member index index_aw1. + */ static TestSecurityConfig.User LIMITED_USER_AB1_ALIAS_READ_ONLY = new TestSecurityConfig.User("limited_user_alias_AB1_read_only")// .description("read/only on alias_ab1w, but with write privs in write index index_aw1")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "write", "indices:admin/refresh") @@ -359,26 +408,12 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_index", limitedToNone())// .indexMatcher("manage_alias", limitedToNone()); - static TestSecurityConfig.User LIMITED_USER_ALIAS_C1 = new TestSecurityConfig.User("limited_user_alias_C1")// - .description("alias_c1")// - .roles( - // - new Role("r1")// - .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "write", "indices_monitor") - .on("alias_c1") - )// - .indexMatcher("read", limitedTo(index_cr1, index_cw1, alias_c1))// - .indexMatcher("write", limitedTo(index_cr1, index_cw1, alias_c1)) // - .indexMatcher("create_index", limitedTo(index_cw1))// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedTo(alias_c1)); - + /** + * A simple test user which has read/only privileges for "*" + */ static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// .description("read/only on *")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read") @@ -391,10 +426,12 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple test user which has read/only privileges for "index_a*" + */ static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// .description("read/only on index_a*")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read") @@ -407,10 +444,12 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple test user that only has index privileges for indices that are not used by this test. + */ static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// .description("no privileges for existing indices")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("crud", "indices_monitor") @@ -423,6 +462,9 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A simple test user that has no index privileges at all. + */ static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// .description("no index privileges")// .roles( @@ -436,156 +478,26 @@ public class IndexAuthorizationReadWriteIntTests { .indexMatcher("manage_alias", limitedToNone())// .indexMatcher("get_alias", limitedToNone()); + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// .roles( - // new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("*") .on("*")// .indexPermissions("*") .on("*") - - )// - .indexMatcher( - "read", - limitedTo( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw2, - index_bwx1, - index_bwx2, - index_cr1, - index_cw1, - alias_ab1w, - alias_ab1r, - alias_c1, - alias_bwx, - alias_ab1w_nowriteindex, - index_hidden - ) - )// - .indexMatcher( - "write", - limitedTo( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw2, - index_bwx1, - index_bwx2, - index_cr1, - index_cw1, - alias_ab1w, - alias_ab1r, - alias_c1, - alias_bwx, - alias_ab1w_nowriteindex, - index_hidden - ) - )// - .indexMatcher( - "create_index", - limitedTo( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw2, - index_bwx1, - index_bwx2, - index_cr1, - index_cw1, - alias_ab1w, - alias_ab1r, - alias_c1, - alias_bwx, - alias_ab1w_nowriteindex, - index_hidden - ) )// - .indexMatcher( - "manage_index", - limitedTo( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw2, - index_bwx1, - index_bwx2, - index_cr1, - index_cw1, - alias_ab1w, - alias_ab1r, - alias_c1, - alias_bwx, - alias_ab1w_nowriteindex, - index_hidden - ) - )// - .indexMatcher( - "manage_alias", - limitedTo( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw2, - index_bwx1, - index_bwx2, - index_cr1, - index_cw1, - alias_ab1w, - alias_ab1r, - alias_c1, - alias_bwx, - alias_ab1w_nowriteindex, - index_hidden - ) - )// - .indexMatcher( - "get_alias", - limitedTo( - index_ar1, - index_ar2, - index_aw1, - index_aw2, - index_br1, - index_br2, - index_bw1, - index_bw2, - index_bwx1, - index_bwx2, - index_cr1, - index_cw1, - alias_ab1w, - alias_ab1r, - alias_c1, - alias_bwx, - alias_ab1w_nowriteindex, - index_hidden - ) - ); + .indexMatcher("read", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("write", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("create_index", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("manage_index", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("manage_alias", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .indexMatcher("get_alias", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx)); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -610,10 +522,8 @@ public class IndexAuthorizationReadWriteIntTests { LIMITED_USER_B_READ_ONLY_MANAGE_INDEX_ALIAS, LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS, LIMITED_USER_B_SYSTEM_INDEX_MANAGE, - LIMITED_USER_C, LIMITED_USER_AB1_ALIAS, LIMITED_USER_AB1_ALIAS_READ_ONLY, - LIMITED_USER_ALIAS_C1, LIMITED_READ_ONLY_ALL, LIMITED_READ_ONLY_A, LIMITED_USER_OTHER_PRIVILEGES, diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index a5c600fae8..bc279c6c95 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -126,6 +126,12 @@ public HttpResponse get(String path, Header... headers) { return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); } + public HttpResponse get(String path, HttpEntity entity, Header... headers) { + HttpGet uriRequest = new HttpGet(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse getWithoutLeadingSlash(String path, Header... headers) { HttpUriRequest req = new HttpGet(getHttpServerUri()); req.setPath(path); @@ -200,6 +206,12 @@ public HttpResponse delete(String path, Header... headers) { return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); } + public HttpResponse delete(String path, HttpEntity entity, Header... headers) { + HttpDelete uriRequest = new HttpDelete(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse postJson(String path, String body, Header... headers) { HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); uriRequest.setEntity(new StringEntity(body)); diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java index 04769faf78..84a3975af4 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java @@ -129,11 +129,11 @@ public static OnResponseIndexMatcher containsExactly(Collection testIndices) { + static OnUserIndexMatcher limitedTo(Collection testIndices) { Map indexNameMap = new HashMap<>(); for (TestIndexOrAliasOrDatastream testIndex : testIndices) { @@ -143,17 +143,24 @@ public static IndexMatcher limitedTo(Collection te return new LimitedToMatcher(indexNameMap); } - public static IndexMatcher unlimited() { + static IndexMatcher unlimited() { return new UnlimitedMatcher(); } - public static IndexMatcher unlimitedIncludingOpenSearchSecurityIndex() { + static IndexMatcher unlimitedIncludingOpenSearchSecurityIndex() { return new UnlimitedMatcher(true); } - public static IndexMatcher limitedToNone() { + static IndexMatcher limitedToNone() { return new LimitedToMatcher(Collections.emptyMap()); } + + /** + * Adds the given indices to the set of indices this matcher is limited to. + * @param testIndices additional indices for the limitation. + * @return a new IndexMatcher instance with the new limit. + */ + OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices); } /** @@ -186,6 +193,10 @@ public interface IndexMatcher extends Matcher { default boolean coversAll(TestIndexOrAliasOrDatastream... testIndices) { return Stream.of(testIndices).allMatch(this::covers); } + + default boolean coversAll(Collection testIndices) { + return testIndices.stream().allMatch(this::covers); + } } // ---------------------------------------------------------------------------------- @@ -678,6 +689,17 @@ protected boolean matchesByIndices( return false; } } + + @Override + public OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices) { + Map indexNameMap = new HashMap<>(this.expectedIndices); + + for (TestIndexOrAliasOrDatastream testIndex : testIndices) { + indexNameMap.put(testIndex.name(), testIndex); + } + + return new LimitedToMatcher(indexNameMap); + } } /** @@ -733,5 +755,10 @@ public int size() { public boolean covers(TestIndexOrAliasOrDatastream testIndex) { return true; } + + @Override + public OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices) { + return this; + } } } From ba1ae0e56f24fc9be1c0322a612c114402a4f2b8 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 3 Oct 2025 05:02:07 +0200 Subject: [PATCH 13/22] Adaption to legacy version Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 7519579a3e..0255149cc6 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -278,7 +278,7 @@ public class IndexAuthorizationReadOnlyIntTests { .on(".system_index_plugin") )// .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index))// - .indexMatcher("get_alias", limitedTo(index_c1, alias_c1, system_index_plugin)); + .indexMatcher("get_alias", limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index)); /** * This user has no privileges for indices that are used in this test. But they have privileges for other indices. @@ -669,19 +669,12 @@ public void search_staticIndices_systemIndex_alias() throws Exception { assertThat(httpResponse, isForbidden()); } } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { - if (user == UNLIMITED_USER) { - // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, - // but withholds documents on the DLS level - assertThat(httpResponse, isOk()); - assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); - } else { assertThat( httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index") .reducedBy(user.indexMatcher("read")) .whenEmpty(isForbidden()) ); - } } else { if (user.indexMatcher("read").covers(alias_with_system_index)) { assertThat(httpResponse, isOk()); @@ -1557,7 +1550,7 @@ public void getAlias_indexPattern_includeHidden() throws Exception { } else { assertThat( httpResponse, - containsExactly(alias_ab1, alias_c1).at("$.*.aliases.keys()") + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()") .reducedBy(user.indexMatcher("get_alias")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1802,25 +1795,13 @@ public void pit_list_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_search/point_in_time/_all"); - if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { - // Once again, the system index privilege code makes it impossible to use this action without super admin privileges - if (user == SUPER_UNLIMITED_USER) { + // At the moment, it is sufficient to have any privileges for any existing index to use the _all API + // This is clearly a bug; yet, not a severe issue, as we do not have really sensitive things available here + if (user != LIMITED_USER_NONE && user != LIMITED_USER_OTHER_PRIVILEGES) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); } - } else { - // The behavior in legacy privilege evaluation and new privilege evaluation actually differs, even though we do not observe - // here a difference: - // - Legacy: the user needs to have the privilege for all indices. If it is only granted for a subset of indices, this will - // be forbidden. - // - New: this is now a cluster privilege, the users below are the users with full cluster privileges - if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { - assertThat(httpResponse, isOk()); - } else { - assertThat(httpResponse, isForbidden()); - } - } } finally { deletePit(indexA1pitId); } From 7601c633708f32563b1cb558b240830cb3751ba3 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 3 Oct 2025 05:18:04 +0200 Subject: [PATCH 14/22] Adaption to legacy version Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 0255149cc6..7e4eefedb6 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -669,12 +669,12 @@ public void search_staticIndices_systemIndex_alias() throws Exception { assertThat(httpResponse, isForbidden()); } } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { - assertThat( - httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(isForbidden()) - ); + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index") + .reducedBy(user.indexMatcher("read")) + .whenEmpty(isForbidden()) + ); } else { if (user.indexMatcher("read").covers(alias_with_system_index)) { assertThat(httpResponse, isOk()); @@ -867,7 +867,7 @@ public void search_alias_pattern_negation() throws Exception { .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); -/* + /* if (user != LIMITED_USER_NONE) { if (clusterConfig.systemIndexPrivilegeEnabled) { // If the system index privilege is enabled, we might also see the system_index_plugin index (being included via the @@ -1448,7 +1448,7 @@ public void getAlias_all() throws Exception { } else { assertThat( httpResponse, - containsExactly(alias_ab1, alias_c1).at("$.*.aliases.keys()") + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()") .reducedBy(user.indexMatcher("get_alias")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1795,13 +1795,16 @@ public void pit_list_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_search/point_in_time/_all"); - // At the moment, it is sufficient to have any privileges for any existing index to use the _all API - // This is clearly a bug; yet, not a severe issue, as we do not have really sensitive things available here - if (user != LIMITED_USER_NONE && user != LIMITED_USER_OTHER_PRIVILEGES) { - assertThat(httpResponse, isOk()); - } else { - assertThat(httpResponse, isForbidden()); - } + // At the moment, it is sufficient to have any privileges for any existing index to use the _all API + // This is clearly a bug; yet, not a severe issue, as we do not have really sensitive things available here. + // This is caused by the following line which makes PrivilegesEvaluator believe it could reduce the indices + // to authorized indices, even though it actually could not: + // https://github.com/opensearch-project/security/blob/aee54a8ca2a6cc596cb1e490be1e9fa240286246/src/main/java/org/opensearch/security/resolver/IndexResolverReplacer.java#L824-L825 + if (user != LIMITED_USER_NONE && user != LIMITED_USER_OTHER_PRIVILEGES) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } } finally { deletePit(indexA1pitId); } @@ -1848,24 +1851,11 @@ public void pit_catSegments_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_cat/pit_segments/_all"); - if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { - // Once again, the system index privilege code makes it impossible to use this action without super admin privileges - if (user == SUPER_UNLIMITED_USER) { - assertThat(httpResponse, isOk()); - } else { - assertThat(httpResponse, isForbidden()); - } + // The user needs to have the privilege for all indices. If it is only granted for a subset of indices, this will be forbidden. + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); } else { - // The behavior in legacy privilege evaluation and new privilege evaluation actually differs, even though we do not observe - // here a difference: - // - Legacy: the user needs to have the privilege for all indices. If it is only granted for a subset of indices, this will - // be forbidden. - // - New: this is now a separate cluster privilege, the users below are the users with full cluster privileges - if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { - assertThat(httpResponse, isOk()); - } else { - assertThat(httpResponse, isForbidden()); - } + assertThat(httpResponse, isForbidden()); } } finally { deletePit(indexA1pitId); From 6fd7108ec0d272c1c13aeb79047009d17ccc79e3 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 3 Oct 2025 06:30:10 +0200 Subject: [PATCH 15/22] Cleanup Signed-off-by: Nils Bandener --- ...taStreamAuthorizationReadOnlyIntTests.java | 57 +++++++++--- ...aStreamAuthorizationReadWriteIntTests.java | 86 ++++++++++++++++--- .../IndexAuthorizationReadWriteIntTests.java | 24 +++--- 3 files changed, 131 insertions(+), 36 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index f957c33429..b101b97ce1 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -58,6 +58,11 @@ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class DataStreamAuthorizationReadOnlyIntTests { + // ------------------------------------------------------------------------------------------------------- + // Test data streams and indices used by this test suite. Indices are usually initially created; the only + // exception is ds_ax, which is referred to in tests, but which does not exist on purpose. + // ------------------------------------------------------------------------------------------------------- + static TestDataStream ds_a1 = TestDataStream.name("ds_a1").documentCount(100).rolloverAfter(10).seed(1).build(); static TestDataStream ds_a2 = TestDataStream.name("ds_a2").documentCount(110).rolloverAfter(10).seed(2).build(); static TestDataStream ds_a3 = TestDataStream.name("ds_a3").documentCount(120).rolloverAfter(10).seed(3).build(); @@ -78,6 +83,18 @@ public class DataStreamAuthorizationReadOnlyIntTests { openSearchSecurityConfigIndex() ); + static final List ALL_DATA_STREAMS = List.of(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from ds_a* + */ static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("ds_a*")// .roles( @@ -94,6 +111,9 @@ public class DataStreamAuthorizationReadOnlyIntTests { )// .indexMatcher("read", limitedTo(ds_a1, ds_a2, ds_a3, ds_ax)); + /** + * A simple user that can read from ds_b* + */ static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("ds_b*")// .roles( @@ -110,6 +130,9 @@ public class DataStreamAuthorizationReadOnlyIntTests { )// .indexMatcher("read", limitedTo(ds_b1, ds_b2, ds_b3)); + /** + * A simple user that can read from ds_b1 + */ static TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// .description("ds_b1")// .roles( @@ -126,7 +149,11 @@ public class DataStreamAuthorizationReadOnlyIntTests { )// .indexMatcher("read", limitedTo(ds_b1)); - static TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + /** + * This user has no privileges for indices that are used in this test. But they have privileges for other indices. + * This allows them to use actions like _search and receive empty result sets. + */ + static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_index_privileges")// .description("no privileges for existing indices")// .roles( new TestSecurityConfig.Role("r1")// @@ -142,6 +169,10 @@ public class DataStreamAuthorizationReadOnlyIntTests { )// .indexMatcher("read", limitedToNone()); + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// .roles( @@ -165,7 +196,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { LIMITED_USER_A, LIMITED_USER_B, LIMITED_USER_B1, - LIMITED_USER_NONE, + LIMITED_USER_OTHER_PRIVILEGES, UNLIMITED_USER, SUPER_UNLIMITED_USER ); @@ -230,7 +261,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + containsExactly(ALL_INDICES).at("hits.hits[*]._index") .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isNotFound() : isForbidden()) ); @@ -243,7 +274,7 @@ public void search_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + containsExactly(ALL_INDICES).at("hits.hits[*]._index") .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -276,7 +307,7 @@ public void search_wildcard() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + containsExactly(ALL_INDICES).at("hits.hits[*]._index") .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -483,7 +514,7 @@ public void search_termsAggregation_index() throws Exception { if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("aggregations.indices.buckets[*].key") + containsExactly(ALL_INDICES).at("aggregations.indices.buckets[*].key") .reducedBy(user.indexMatcher("read")) .whenEmpty(isOk()) ); @@ -520,7 +551,7 @@ public void index_stats_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_stats"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("indices.keys()") + containsExactly(ALL_INDICES).at("indices.keys()") .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -547,7 +578,7 @@ public void getDataStream_all() throws Exception { // The legacy mode does not support dnfof for indices:admin/data_stream/get assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].name") + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name") .butForbiddenIfIncomplete(user.indexMatcher("read")) ); } @@ -560,7 +591,7 @@ public void getDataStream_wildcard() throws Exception { // The legacy mode does not support dnfof for indices:admin/data_stream/get assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].name") + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name") .butForbiddenIfIncomplete(user.indexMatcher("read")) ); } @@ -608,7 +639,7 @@ public void getDataStreamStats_all() throws Exception { // The legacy mode does not support dnfof for indices:monitor/data_stream/stats assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].data_stream") + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream") .butForbiddenIfIncomplete(user.indexMatcher("read")) ); } @@ -621,7 +652,7 @@ public void getDataStreamStats_wildcard() throws Exception { // The legacy mode does not support dnfof for indices:monitor/data_stream/stats assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.data_streams[*].data_stream") + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream") .butForbiddenIfIncomplete(user.indexMatcher("read")) ); } @@ -656,7 +687,7 @@ public void resolve_wildcard() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("$.*[*].name") + containsExactly(ALL_INDICES).at("$.*[*].name") .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -682,7 +713,7 @@ public void field_caps_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("indices") + containsExactly(ALL_INDICES).at("indices") .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java index 05ab55d444..ebd899fdeb 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java @@ -20,10 +20,12 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.collect.ImmutableList; +import org.junit.After; import org.junit.AfterClass; import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.test.framework.TestComponentTemplate; import org.opensearch.test.framework.TestDataStream; import org.opensearch.test.framework.TestIndex; @@ -64,6 +66,14 @@ @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @NotThreadSafe public class DataStreamAuthorizationReadWriteIntTests { + + // ------------------------------------------------------------------------------------------------------- + // Test indices used by this test suite. We use the following naming scheme: + // - index_*r*, ds_*r*: This test will not write to this index or data stream + // - index_*w*, ds_*w*: This test can write to this index or data stream; the test won't delete and recreate it + // - index_*wx*, ds_*wx*: The index is not initially created; the test can create it on demand and delete it again + // ------------------------------------------------------------------------------------------------------- + static TestDataStream ds_ar1 = TestDataStream.name("ds_ar1").documentCount(22).rolloverAfter(10).build(); static TestDataStream ds_ar2 = TestDataStream.name("ds_ar2").documentCount(22).rolloverAfter(10).build(); static TestDataStream ds_aw1 = TestDataStream.name("ds_aw1").documentCount(22).rolloverAfter(10).build(); @@ -79,12 +89,22 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestDataStream ds_bwx1 = TestDataStream.name("ds_bwx1").documentCount(0).build(); // not initially created static TestDataStream ds_bwx2 = TestDataStream.name("ds_bwx2").documentCount(0).build(); // not initially created + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from ds_a* and write to ds_aw*; the user as no privileges to create or manage data streams + */ static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("ds_a*")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_a*")// .indexPermissions("write") .on("ds_aw*") @@ -94,12 +114,15 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple user that can read from ds_b* and write to ds_bw*; the user as no privileges to create or manage data streams + */ static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("ds_b*")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_b*")// .indexPermissions("write") .on("ds_bw*") @@ -109,12 +132,16 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple user that can read from ds_b* and write to ds_bw*; the user as no privileges to create or manage data streams. + * Additionally, they can read from ds_a* + */ static TestSecurityConfig.User LIMITED_USER_B_READ_ONLY_A = new TestSecurityConfig.User("limited_user_B_read_only_A")// .description("ds_b*; read only on ds_a*")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_a*", "ds_b*")// .indexPermissions("write") .on("ds_bw*") @@ -135,7 +162,7 @@ public class DataStreamAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_b*")// .indexPermissions("write") .on("ds_bw*")// @@ -147,12 +174,15 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple user that can read from ds_b* and write to ds_bw*; they can also create data streams with the name ds_bw* + */ static TestSecurityConfig.User LIMITED_USER_B_CREATE_DS = new TestSecurityConfig.User("limited_user_B_create_ds")// .description("ds_b* with create ds privs")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_b*")// .indexPermissions("write") .on("ds_bw*")// @@ -164,12 +194,15 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple user that can read from ds_b* and write to ds_bw*; they can also create and manage data streams with the name ds_bw* + */ static TestSecurityConfig.User LIMITED_USER_B_MANAGE_DS = new TestSecurityConfig.User("limited_user_B_manage_ds")// .description("ds_b* with manage privs")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_b*")// .indexPermissions("write") .on("ds_bw*")// @@ -181,12 +214,15 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// .indexMatcher("manage_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + /** + * A user that can read from ds_a* and ds_b* and write/create/manage ds_aw*, ds_bw* + */ static TestSecurityConfig.User LIMITED_USER_AB_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_AB_manage_index")// .description("ds_a*, ds_b* with manage index privs")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("ds_a*", "ds_b*")// .indexPermissions("write") .on("ds_aw*", "ds_bw*")// @@ -198,12 +234,15 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// .indexMatcher("manage_data_stream", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + /** + * A simple user that can read from index_c* + */ static TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// .description("index_c*")// .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh") + .indexPermissions("read", "indices_monitor") .on("index_c*")// .indexPermissions("write") .on("index_cw*") @@ -213,6 +252,9 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple user that can read all indices and data streams, but cannot write anything + */ static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// .description("read/only on *")// .roles( @@ -226,6 +268,9 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple user that can read from ds_a*, but cannot write anything + */ static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// .description("read/only on ds_a*")// .roles( @@ -239,6 +284,9 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple test user that only has index privileges for indices that are not used by this test. + */ static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// .description("no privileges for existing indices")// .roles( @@ -252,6 +300,9 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A simple test user that has no index privileges at all. + */ static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// .description("no index privileges")// .roles( @@ -263,6 +314,9 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * This user has only privileges on backing indices for data streams, but not on the data streams themselves + */ static TestSecurityConfig.User LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES = new TestSecurityConfig.User( "limited_user_permissions_on_backing_indices" )// @@ -270,7 +324,7 @@ public class DataStreamAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on(".ds-ds_a*")// .indexPermissions("write") .on(".ds-ds_aw*") @@ -280,6 +334,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexMatcher("create_data_stream", limitedToNone())// .indexMatcher("manage_data_stream", limitedToNone()); + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// .roles( @@ -381,7 +439,7 @@ public void deleteByQuery_indexPattern() throws Exception { assertThat(httpResponse, isCreated()); } - HttpResponse httpResponse = restClient.postJson("ds_aw*,ds_bw*/_delete_by_query?refresh=true&wait_for_completion=true", """ + HttpResponse httpResponse = restClient.postJson("ds_aw*,ds_bw*/_delete_by_query?wait_for_completion=true", """ { "query": { "term": { @@ -415,7 +473,7 @@ public void deleteByQuery_indexPattern() throws Exception { @Test public void putDocument_bulk() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - HttpResponse httpResponse = restClient.putJson("_bulk?refresh=true", """ + HttpResponse httpResponse = restClient.putJson("_bulk", """ { "create": { "_index": "ds_aw1", "_id": "d1" } } { "a": 1, "test": "putDocument_bulk", "@timestamp": "2025-09-15T12:00:00Z" } { "create": { "_index": "ds_bw1", "_id": "d1" } } @@ -494,6 +552,11 @@ public void deleteDataStream() throws Exception { } } + @After + public void refresh() { + cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); + } + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") public static Collection params() { List result = new ArrayList<>(); @@ -523,6 +586,7 @@ private void delete(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastream private void deleteTestDocs(String testName, String indices) { try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + adminRestClient.post(indices + "/_refresh"); adminRestClient.postJson(indices + "/_delete_by_query?refresh=true&wait_for_completion=true", """ { "query": { diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index d7a6760f4f..d3fdab8400 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -150,7 +150,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_a*")// .indexPermissions("write") .on("index_aw*") @@ -170,7 +170,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_b*")// .indexPermissions("write") .on("index_bw*") @@ -190,7 +190,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_b*")// .indexPermissions("write") .on("index_bw*")// @@ -212,7 +212,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_b*")// .indexPermissions("write") .on("index_bw*")// @@ -236,7 +236,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_b*")// .indexPermissions("write") .on("index_bw*")// @@ -264,7 +264,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_b*")// .indexPermissions("crud", "manage", "manage_aliases") .on("alias_bwx*") @@ -286,7 +286,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor") .on("index_a*", "index_b*", "index_hidden*")// .indexPermissions("write") .on("index_bw*", "index_hidden*")// @@ -326,7 +326,7 @@ public class IndexAuthorizationReadWriteIntTests { .roles( new Role("r1")// .clusterPermissions("cluster_composite_ops", "cluster_monitor")// - .indexPermissions("read", "indices_monitor", "indices:admin/refresh*", "system:admin/system_index") + .indexPermissions("read", "indices_monitor", "system:admin/system_index") .on("index_b*", "index_hidden*", ".system_index_plugin")// .indexPermissions("write", "system:admin/system_index") .on("index_bw*", ".system_index_plugin", ".system_index_plugin_*")// @@ -375,7 +375,7 @@ public class IndexAuthorizationReadWriteIntTests { .clusterPermissions("cluster_composite_ops", "cluster_monitor")// .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get") .on("alias_ab1r")// - .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get", "write", "indices:admin/refresh*") + .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get", "write") .on("alias_ab1w*") )// .indexMatcher( @@ -640,7 +640,7 @@ public void deleteByQuery_indexPattern() throws Exception { ); assertThat(httpResponse, isCreated()); - httpResponse = restClient.postJson("index_aw*,index_bw*/_delete_by_query?refresh=true&wait_for_completion=true", """ + httpResponse = restClient.postJson("index_aw*,index_bw*/_delete_by_query?wait_for_completion=true", """ { "query": { "term": { @@ -673,7 +673,7 @@ public void putDocument_bulk() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { IndexApiResponseMatchers.IndexMatcher writePrivileges = user.indexMatcher("write"); - HttpResponse httpResponse = restClient.putJson("_bulk?refresh=true", """ + HttpResponse httpResponse = restClient.putJson("_bulk", """ {"index": {"_index": "index_aw1", "_id": "new_doc_aw1"}} {"a": 1} {"index": {"_index": "index_bw1", "_id": "new_doc_bw1"}} @@ -728,7 +728,7 @@ public void putDocument_alias_noWriteIndex() throws Exception { @Test public void putDocument_bulk_alias() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - HttpResponse httpResponse = restClient.putJson("_bulk?refresh=true", """ + HttpResponse httpResponse = restClient.putJson("_bulk", """ {"index": {"_index": "alias_ab1w", "_id": "put_doc_alias_bulk_test_1"}} {"a": 1} """); From e3cc1133865b5d282ca4f895889a563fc5624141 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 3 Oct 2025 06:42:52 +0200 Subject: [PATCH 16/22] Cleanup Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 22 ------------------- .../IndexAuthorizationReadWriteIntTests.java | 10 --------- 2 files changed, 32 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 7e4eefedb6..95ededb72d 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -867,28 +867,6 @@ public void search_alias_pattern_negation() throws Exception { .reducedBy(user.indexMatcher("read")) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); - /* - if (user != LIMITED_USER_NONE) { - if (clusterConfig.systemIndexPrivilegeEnabled) { - // If the system index privilege is enabled, we might also see the system_index_plugin index (being included via the - // alias) - assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) - ); - } else { - assertThat( - httpResponse, - containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) - ); - } - } else { - assertThat(httpResponse, isForbidden()); - }*/ } } diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index d3fdab8400..5b87657ab2 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -748,7 +748,6 @@ public void putDocument_bulk_alias() throws Exception { } else { assertThat(httpResponse, isForbidden()); } - } finally { delete("index_aw1/_doc/put_doc_alias_bulk_test_1"); } @@ -868,7 +867,6 @@ public void deleteAlias_staticIndex() throws Exception { assertThat(httpResponse, isForbidden()); } } - } finally { delete(alias_bwx); } @@ -1118,20 +1116,12 @@ public void rollover_explicitTargetIndex() throws Exception { } }"""); - System.out.println(httpResponse.getBody()); - if (clusterConfig.legacyPrivilegeEvaluation) { if (user.indexMatcher("manage_alias").covers(index_bw1) && user.indexMatcher("manage_index").covers(index_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); } - } else { - if (user.indexMatcher("manage_alias").covers(alias_bwx) && user.indexMatcher("manage_index").covers(index_bw2)) { - assertThat(httpResponse, isOk()); - } else { - assertThat(httpResponse, isForbidden()); - } } } finally { delete(alias_bwx, index_bwx1); From 2c0a7011f4e20426fd1a634029a11945f7b47b3c Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 7 Oct 2025 10:53:10 +0200 Subject: [PATCH 17/22] Moved user.indexMatcher() to user.reference() Signed-off-by: Nils Bandener --- ...taStreamAuthorizationReadOnlyIntTests.java | 111 ++++--- ...aStreamAuthorizationReadWriteIntTests.java | 172 +++++----- .../IndexAuthorizationReadOnlyIntTests.java | 256 ++++++++------- .../IndexAuthorizationReadWriteIntTests.java | 296 +++++++++--------- .../SnapshotAuthorizationIntTests.java | 81 ++--- .../test/framework/TestDataStream.java | 5 + .../opensearch/test/framework/TestIndex.java | 2 +- .../test/framework/TestSecurityConfig.java | 25 +- ...seMatchers.java => RestIndexMatchers.java} | 2 +- 9 files changed, 507 insertions(+), 443 deletions(-) rename src/integrationTest/java/org/opensearch/test/framework/matcher/{IndexApiResponseMatchers.java => RestIndexMatchers.java} (99%) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index b101b97ce1..3de08d4832 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -29,16 +29,17 @@ import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; import static org.opensearch.test.framework.matcher.RestMatchers.isOk; @@ -85,6 +86,14 @@ public class DataStreamAuthorizationReadOnlyIntTests { static final List ALL_DATA_STREAMS = List.of(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3); + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + // ------------------------------------------------------------------------------------------------------- // Test users with which the tests will be executed; the users need to be added to the list USERS below // The users have two redundant versions or privilege configuration, which needs to be kept in sync: @@ -109,7 +118,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_a*") )// - .indexMatcher("read", limitedTo(ds_a1, ds_a2, ds_a3, ds_ax)); + .reference(READ, limitedTo(ds_a1, ds_a2, ds_a3, ds_ax)); /** * A simple user that can read from ds_b* @@ -128,7 +137,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_b*") )// - .indexMatcher("read", limitedTo(ds_b1, ds_b2, ds_b3)); + .reference(READ, limitedTo(ds_b1, ds_b2, ds_b3)); /** * A simple user that can read from ds_b1 @@ -147,7 +156,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_b1") )// - .indexMatcher("read", limitedTo(ds_b1)); + .reference(READ, limitedTo(ds_b1)); /** * This user has no privileges for indices that are used in this test. But they have privileges for other indices. @@ -167,7 +176,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { ) .on("ds_does_not_exist_*") )// - .indexMatcher("read", limitedToNone()); + .reference(READ, limitedToNone()); /** * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index @@ -181,7 +190,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { .indexPermissions("*") .on("*")// )// - .indexMatcher("read", unlimited()); + .reference(READ, unlimited()); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -190,7 +199,7 @@ public class DataStreamAuthorizationReadOnlyIntTests { static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// .description("super unlimited (admin cert)")// .adminCertUser()// - .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex()); + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex()); static List USERS = List.of( LIMITED_USER_A, @@ -228,7 +237,7 @@ public void search_noPattern() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -247,7 +256,7 @@ public void search_noPattern_noWildcards() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -262,7 +271,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isNotFound() : isForbidden()) ); } @@ -275,7 +284,7 @@ public void search_all() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -294,7 +303,7 @@ public void search_all_noWildcards() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -308,7 +317,7 @@ public void search_wildcard() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -321,7 +330,7 @@ public void search_staticNames_noIgnoreUnavailable() throws Exception { // With dnfof data streams with incomplete privileges will be replaced by their member indices assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -333,7 +342,7 @@ public void search_staticNames_ignoreUnavailable() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -346,7 +355,7 @@ public void search_staticIndicies_negation_backingIndices() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -359,7 +368,7 @@ public void search_indexPattern() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -375,7 +384,7 @@ public void search_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -396,7 +405,7 @@ public void search_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -412,7 +421,7 @@ public void search_indexPattern_minus_backingIndices() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -421,7 +430,7 @@ public void search_indexPattern_minus_backingIndices() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -444,7 +453,7 @@ public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exce assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -465,7 +474,7 @@ public void search_indexPattern_noWildcards() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -515,7 +524,7 @@ public void search_termsAggregation_index() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("aggregations.indices.buckets[*].key") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isOk()) ); } else { @@ -540,7 +549,7 @@ public void msearch_staticIndices() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.postJson("_msearch", msearchBody); assertThat( httpResponse, - containsExactly(ds_b1, ds_b2).at("responses[*].hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isOk()) + containsExactly(ds_b1, ds_b2).at("responses[*].hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isOk()) ); } } @@ -552,7 +561,7 @@ public void index_stats_all() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("indices.keys()") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -565,7 +574,7 @@ public void index_stats_pattern() throws Exception { assertThat( httpResponse, containsExactly(ds_b1, ds_b2, ds_b3).at("indices.keys()") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -579,7 +588,7 @@ public void getDataStream_all() throws Exception { assertThat( httpResponse, containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name") - .butForbiddenIfIncomplete(user.indexMatcher("read")) + .butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -592,7 +601,7 @@ public void getDataStream_wildcard() throws Exception { assertThat( httpResponse, containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name") - .butForbiddenIfIncomplete(user.indexMatcher("read")) + .butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -604,7 +613,7 @@ public void getDataStream_pattern() throws Exception { // The legacy mode does not support dnfof for indices:admin/data_stream/get assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -616,7 +625,7 @@ public void getDataStream_pattern_negation() throws Exception { // The legacy mode does not support dnfof for indices:admin/data_stream/get assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -627,7 +636,7 @@ public void getDataStream_static() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a1,ds_a2"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(ds_a1, ds_a2).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -640,7 +649,7 @@ public void getDataStreamStats_all() throws Exception { assertThat( httpResponse, containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream") - .butForbiddenIfIncomplete(user.indexMatcher("read")) + .butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -653,7 +662,7 @@ public void getDataStreamStats_wildcard() throws Exception { assertThat( httpResponse, containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream") - .butForbiddenIfIncomplete(user.indexMatcher("read")) + .butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -665,7 +674,7 @@ public void getDataStreamStats_pattern() throws Exception { // The legacy mode does not support dnfof for indices:monitor/data_stream/stats assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -676,7 +685,7 @@ public void getDataStreamStats_static() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a1,ds_a2/_stats"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(ds_a1, ds_a2).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -688,7 +697,7 @@ public void resolve_wildcard() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("$.*[*].name") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -701,7 +710,7 @@ public void resolve_indexPattern() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.*[*].name") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -714,7 +723,7 @@ public void field_caps_all() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -727,7 +736,7 @@ public void field_caps_indexPattern() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -739,7 +748,7 @@ public void field_caps_staticIndices_noIgnoreUnavailable() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_field_caps?fields=*"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_b1).at("indices").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(ds_a1, ds_a2, ds_b1).at("indices").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } @@ -752,7 +761,7 @@ public void field_caps_staticIndices_ignoreUnavailable() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_b1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -790,7 +799,7 @@ public void field_caps_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -801,7 +810,7 @@ public void field_caps_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -817,7 +826,7 @@ public void field_caps_indexPattern_minus_backingIndices() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -825,7 +834,7 @@ public void field_caps_indexPattern_minus_backingIndices() throws Exception { assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -839,7 +848,7 @@ public void field_caps_staticIndices_negation_backingIndices() throws Exception assertThat( httpResponse, containsExactly(ds_a1, ds_a2, ds_b1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java index ebd899fdeb..ac6b850b8e 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java @@ -36,15 +36,16 @@ import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.cluster.TestRestClient.json; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; @@ -89,6 +90,35 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestDataStream ds_bwx1 = TestDataStream.name("ds_bwx1").documentCount(0).build(); // not initially created static TestDataStream ds_bwx2 = TestDataStream.name("ds_bwx2").documentCount(0).build(); // not initially created + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index write permissions of individual users. This does + * not include index creation permissions. + */ + static final TestSecurityConfig.User.MetadataKey WRITE = new TestSecurityConfig.User.MetadataKey<>( + "write", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for create index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey CREATE_DATA_STREAM = + new TestSecurityConfig.User.MetadataKey<>("create_data_stream", RestIndexMatchers.IndexMatcher.class); + + /** + * This key identifies assertion reference data for manage index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey MANAGE_DATA_STREAM = + new TestSecurityConfig.User.MetadataKey<>("manage_data_stream", RestIndexMatchers.IndexMatcher.class); + // ------------------------------------------------------------------------------------------------------- // Test users with which the tests will be executed; the users need to be added to the list USERS below // The users have two redundant versions or privilege configuration, which needs to be kept in sync: @@ -109,10 +139,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("write") .on("ds_aw*") )// - .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// - .indexMatcher("write", limitedTo(ds_aw1, ds_aw2))// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .reference(WRITE, limitedTo(ds_aw1, ds_aw2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple user that can read from ds_b* and write to ds_bw*; the user as no privileges to create or manage data streams @@ -127,10 +157,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("write") .on("ds_bw*") )// - .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple user that can read from ds_b* and write to ds_bw*; the user as no privileges to create or manage data streams. @@ -146,10 +176,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("write") .on("ds_bw*") )// - .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * This is an artificial user - in the sense that in real life it would likely not exist this way. @@ -169,10 +199,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("indices:admin/mapping/auto_put") .on("*") )// - .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple user that can read from ds_b* and write to ds_bw*; they can also create data streams with the name ds_bw* @@ -189,10 +219,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("indices:admin/data_stream/create") .on("ds_bw*") )// - .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("create_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple user that can read from ds_b* and write to ds_bw*; they can also create and manage data streams with the name ds_bw* @@ -209,10 +239,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("manage") .on("ds_bw*") )// - .indexMatcher("read", limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("write", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("create_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("manage_data_stream", limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(MANAGE_DATA_STREAM, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); /** * A user that can read from ds_a* and ds_b* and write/create/manage ds_aw*, ds_bw* @@ -229,10 +259,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("manage") .on("ds_aw*", "ds_bw*") )// - .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("write", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("create_data_stream", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// - .indexMatcher("manage_data_stream", limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(MANAGE_DATA_STREAM, limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); /** * A simple user that can read from index_c* @@ -247,10 +277,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("write") .on("index_cw*") )// - .indexMatcher("read", limitedTo(index_cr1, index_cw1))// - .indexMatcher("write", limitedTo(index_cw1))// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(index_cr1, index_cw1))// + .reference(WRITE, limitedTo(index_cw1))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple user that can read all indices and data streams, but cannot write anything @@ -263,10 +293,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("read") .on("*") )// - .indexMatcher("read", unlimited())// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, unlimited())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple user that can read from ds_a*, but cannot write anything @@ -279,10 +309,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("read") .on("ds_a*") )// - .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple test user that only has index privileges for indices that are not used by this test. @@ -295,10 +325,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("crud", "indices_monitor") .on("ds_does_not_exist_*") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A simple test user that has no index privileges at all. @@ -309,10 +339,10 @@ public class DataStreamAuthorizationReadWriteIntTests { new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * This user has only privileges on backing indices for data streams, but not on the data streams themselves @@ -329,10 +359,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("write") .on(".ds-ds_aw*") )// - .indexMatcher("read", limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// - .indexMatcher("write", limitedTo(ds_aw1, ds_aw2))// - .indexMatcher("create_data_stream", limitedToNone())// - .indexMatcher("manage_data_stream", limitedToNone()); + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .reference(WRITE, limitedTo(ds_aw1, ds_aw2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); /** * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index @@ -346,10 +376,10 @@ public class DataStreamAuthorizationReadWriteIntTests { .indexPermissions("*") .on("*") )// - .indexMatcher("read", unlimited())// - .indexMatcher("write", unlimited())// - .indexMatcher("create_data_stream", unlimited())// - .indexMatcher("manage_data_stream", unlimited()); + .reference(READ, unlimited())// + .reference(WRITE, unlimited())// + .reference(CREATE_DATA_STREAM, unlimited())// + .reference(MANAGE_DATA_STREAM, unlimited()); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -358,10 +388,10 @@ public class DataStreamAuthorizationReadWriteIntTests { static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// .description("super unlimited (admin cert)")// .adminCertUser()// - .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("write", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("create_data_stream", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("manage_data_stream", unlimitedIncludingOpenSearchSecurityIndex()); + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(WRITE, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(CREATE_DATA_STREAM, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(MANAGE_DATA_STREAM, unlimitedIncludingOpenSearchSecurityIndex()); static List USERS = ImmutableList.of( LIMITED_USER_A, @@ -406,7 +436,7 @@ public static void stopClusters() { public void createDocument() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { HttpResponse httpResponse = restClient.post("ds_bw1/_doc/", json("a", 1, "@timestamp", Instant.now().toString())); - assertThat(httpResponse, containsExactly(ds_bw1).at("_index").reducedBy(user.indexMatcher("write")).whenEmpty(isForbidden())); + assertThat(httpResponse, containsExactly(ds_bw1).at("_index").reducedBy(user.reference(WRITE)).whenEmpty(isForbidden())); } } @@ -451,7 +481,7 @@ public void deleteByQuery_indexPattern() throws Exception { if (clusterConfig.legacyPrivilegeEvaluation) { // dnfof is not applicable to indices:data/write/delete/byquery, so we need privileges for all indices - if (user.indexMatcher("write").coversAll(ds_aw1, ds_aw2, ds_bw1, ds_bw2)) { + if (user.reference(WRITE).coversAll(ds_aw1, ds_aw2, ds_bw1, ds_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -459,7 +489,7 @@ public void deleteByQuery_indexPattern() throws Exception { } else { if (user != LIMITED_USER_NONE && user != LIMITED_READ_ONLY_ALL && user != LIMITED_READ_ONLY_A) { assertThat(httpResponse, isOk()); - int expectedDeleteCount = containsExactly(ds_aw1, ds_bw1).at("_index").reducedBy(user.indexMatcher("write")).size(); + int expectedDeleteCount = containsExactly(ds_aw1, ds_bw1).at("_index").reducedBy(user.reference(WRITE)).size(); assertEquals(httpResponse.getBody(), expectedDeleteCount, httpResponse.bodyAsMap().get("deleted")); } else { assertThat(httpResponse, isForbidden()); @@ -490,7 +520,7 @@ public void putDocument_bulk() throws Exception { assertThat( httpResponse, containsExactly(ds_aw1, ds_bw1).at("items[*].create[?(@.result == 'created')]._index") - .reducedBy(user.indexMatcher("write")) + .reducedBy(user.reference(WRITE)) .whenEmpty(isOk()) ); } else { @@ -506,7 +536,7 @@ public void createDataStream() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { HttpResponse httpResponse = restClient.put("_data_stream/ds_bwx1"); - if (containsExactly(ds_bwx1).reducedBy(user.indexMatcher("create_data_stream")).isEmpty()) { + if (containsExactly(ds_bwx1).reducedBy(user.reference(CREATE_DATA_STREAM)).isEmpty()) { assertThat(httpResponse, isForbidden()); } else { assertThat(httpResponse, isOk()); @@ -542,7 +572,7 @@ public void deleteDataStream() throws Exception { HttpResponse httpResponse = restClient.delete("_data_stream/ds_bwx1"); - if (user.indexMatcher("manage_data_stream").isEmpty()) { + if (user.reference(MANAGE_DATA_STREAM).isEmpty()) { assertThat(httpResponse, isForbidden()); } else { assertThat(httpResponse, isOk()); diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index 95ededb72d..f6181c475c 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -37,7 +37,7 @@ import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; -import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; @@ -45,11 +45,11 @@ import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.cluster.TestRestClient.json; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.IndexMatcher; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.IndexMatcher; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; @@ -149,6 +149,32 @@ public class IndexAuthorizationReadOnlyIntTests { index_hidden_dot ); + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + * This reference data generally applies for both legacy privilege evaluation and new privilege evaluation. + * There might be however deviations in some cases; then the READ_NEXT_GEN matcher can be used. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index search/read permissions of individual users for the new privilege evaluation + */ + static final TestSecurityConfig.User.MetadataKey READ_NEXT_GEN = new TestSecurityConfig.User.MetadataKey<>( + "read_nextgen", + IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for operations getting alias meta data of individual users + */ + static final TestSecurityConfig.User.MetadataKey GET_ALIAS = new TestSecurityConfig.User.MetadataKey<>( + "get_alias", + IndexMatcher.class + ); + // ------------------------------------------------------------------------------------------------------- // Test users with which the tests will be executed; the users need to be added to the list USERS below // The users have two redundant versions or privilege configuration, which needs to be kept in sync: @@ -167,8 +193,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze") .on("index_a*") )// - .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax))// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_a1, index_a2, index_a3, index_ax))// + .reference(GET_ALIAS, limitedToNone()); /** * A simple user that can read from index_b* @@ -181,8 +207,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze") .on("index_b*") )// - .indexMatcher("read", limitedTo(index_b1, index_b2, index_b3))// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_b1, index_b2, index_b3))// + .reference(GET_ALIAS, limitedToNone()); /** * A simple user that can read only from index_b1 @@ -195,8 +221,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze") .on("index_b1") )// - .indexMatcher("read", limitedTo(index_b1))// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_b1))// + .reference(GET_ALIAS, limitedToNone()); /** * A simple user that can read from index_c* @@ -209,8 +235,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze") .on("index_c*") )// - .indexMatcher("read", limitedTo(index_c1, alias_c1))// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_c1, alias_c1))// + .reference(GET_ALIAS, limitedToNone()); /** * A user that has read privileges for alias_ab1*; these privileges are inherited to the member indices. @@ -224,8 +250,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") .on("alias_ab1*") )// - .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// - .indexMatcher("get_alias", limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1)); + .reference(READ, limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// + .reference(GET_ALIAS, limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1)); /** * A user that has read privileges for alias_c1; these privileges are inherited to the member indices. @@ -239,8 +265,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") .on("alias_c1") )// - .indexMatcher("read", limitedTo(index_c1, alias_c1))// - .indexMatcher("get_alias", limitedTo(index_c1, alias_c1)); + .reference(READ, limitedTo(index_c1, alias_c1))// + .reference(GET_ALIAS, limitedTo(index_c1, alias_c1)); /** * Same as LIMITED_USER_A with the addition of read privileges for index_hidden* and .index_hidden* */ @@ -252,8 +278,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/analyze") .on("index_a*", "index_hidden*", ".index_hidden*") )// - .indexMatcher("read", limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// + .reference(GET_ALIAS, limitedToNone()); /** * Same as LIMITED_USER_C with the addition of read privileges for ".system_index_plugin"; they also have the @@ -277,8 +303,8 @@ public class IndexAuthorizationReadOnlyIntTests { ) .on(".system_index_plugin") )// - .indexMatcher("read", limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index))// - .indexMatcher("get_alias", limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index)); + .reference(READ, limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index))// + .reference(GET_ALIAS, limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index)); /** * This user has no privileges for indices that are used in this test. But they have privileges for other indices. @@ -294,8 +320,8 @@ public class IndexAuthorizationReadOnlyIntTests { .indexPermissions("crud", "indices_monitor", "indices:admin/analyze") .on("index_does_not_exist_*") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(GET_ALIAS, limitedToNone()); /** * This user has no index read privileges at all. @@ -306,8 +332,8 @@ public class IndexAuthorizationReadOnlyIntTests { new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(GET_ALIAS, limitedToNone()); /** * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index @@ -322,8 +348,8 @@ public class IndexAuthorizationReadOnlyIntTests { .on("*")// )// - .indexMatcher("read", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax))// - .indexMatcher("get_alias", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax)); + .reference(READ, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax))// + .reference(GET_ALIAS, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax)); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -332,8 +358,8 @@ public class IndexAuthorizationReadOnlyIntTests { static final TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// .description("super unlimited (admin cert)")// .adminCertUser()// - .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(GET_ALIAS, unlimitedIncludingOpenSearchSecurityIndex()); static final List USERS = ImmutableList.of( LIMITED_USER_A, @@ -390,7 +416,7 @@ public void search_noPattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -399,7 +425,7 @@ public void search_noPattern() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -421,7 +447,7 @@ public void search_noPattern_noWildcards() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -436,7 +462,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isForbidden()) ); } else { @@ -445,7 +471,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -460,7 +486,7 @@ public void search_all() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -469,7 +495,7 @@ public void search_all() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -491,7 +517,7 @@ public void search_all_noWildcards() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -509,7 +535,7 @@ public void search_all_includeHidden() throws Exception { ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES ).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -524,7 +550,7 @@ public void search_wildcard() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -533,7 +559,7 @@ public void search_wildcard() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -555,7 +581,7 @@ public void search_wildcard_noWildcards() throws Exception { httpResponse, containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( "hits.hits[*]._index" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } } @@ -573,7 +599,7 @@ public void search_wildcard_includeHidden() throws Exception { ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES ).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -585,7 +611,7 @@ public void search_staticIndices() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_search?size=1000"); assertThat( httpResponse, - containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -598,7 +624,7 @@ public void search_staticIndices_ignoreUnavailable() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -609,7 +635,7 @@ public void search_staticIndices_nonExisting() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_search?size=1000"); - if (containsExactly(index_ax).reducedBy(user.indexMatcher("read")).isEmpty()) { + if (containsExactly(index_ax).reducedBy(user.reference(READ)).isEmpty()) { assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); } else { assertThat(httpResponse, isNotFound()); @@ -623,7 +649,7 @@ public void search_staticIndices_hidden() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_search?size=1000"); assertThat( httpResponse, - containsExactly(index_hidden).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(index_hidden).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -635,7 +661,7 @@ public void search_staticIndices_systemIndex() throws Exception { if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { assertThat( httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.indexMatcher("read")) + containsExactly(system_index_plugin).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.reference(READ)) ); } else { // legacy privilege evaluation without system index privilege enabled @@ -672,11 +698,11 @@ public void search_staticIndices_systemIndex_alias() throws Exception { assertThat( httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isForbidden()) ); } else { - if (user.indexMatcher("read").covers(alias_with_system_index)) { + if (user.reference(READ).covers(alias_with_system_index)) { assertThat(httpResponse, isOk()); assertThat(httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index")); } else { @@ -694,7 +720,7 @@ public void search_indexPattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -708,7 +734,7 @@ public void search_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -730,7 +756,7 @@ public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exce assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -751,7 +777,7 @@ public void search_indexPattern_noWildcards() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -768,7 +794,7 @@ public void search_indexPatternAndStatic_negation() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -795,7 +821,7 @@ public void search_indexPattern_includeHidden() throws Exception { index_hidden_dot, system_index_plugin ).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else if (!clusterConfig.systemIndexPrivilegeEnabled) { @@ -804,7 +830,7 @@ public void search_indexPattern_includeHidden() throws Exception { httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, index_hidden, index_hidden_dot) .at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -836,7 +862,7 @@ public void search_alias() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isForbidden()) ); } @@ -851,7 +877,7 @@ public void search_alias_pattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -864,7 +890,7 @@ public void search_alias_pattern_negation() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -886,7 +912,7 @@ public void search_alias_pattern_includeHidden() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { @@ -898,7 +924,7 @@ public void search_alias_pattern_includeHidden() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isOk()) ); } @@ -917,7 +943,7 @@ public void search_aliasAndIndex_ignoreUnavailable() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isForbidden()) ); @@ -966,7 +992,7 @@ public void search_termsAggregation_index() throws Exception { httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at( "aggregations.indices.buckets[*].key" - ).reducedBy(user.indexMatcher("read")).whenEmpty(isOk()) + ).reducedBy(user.reference(READ)).whenEmpty(isOk()) ); } } @@ -976,7 +1002,7 @@ public void search_pit() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.post("index_a*,index_b*/_search/point_in_time?keep_alive=1m"); - IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher = containsExactly( + RestIndexMatchers.OnResponseIndexMatcher indexMatcher = containsExactly( index_a1, index_a2, index_a3, @@ -985,7 +1011,7 @@ public void search_pit() throws Exception { index_b3 ); - if (indexMatcher.reducedBy(user.indexMatcher("read")).isEmpty()) { + if (indexMatcher.reducedBy(user.reference(READ)).isEmpty()) { assertThat( httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") @@ -1001,7 +1027,7 @@ public void search_pit() throws Exception { } """, pitId)); assertThat(httpResponse, isOk()); - assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("read"))); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.reference(READ))); } } } @@ -1011,7 +1037,7 @@ public void search_pit_all() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.post("_all/_search/point_in_time?keep_alive=1m"); - IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher; + RestIndexMatchers.OnResponseIndexMatcher indexMatcher; if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { indexMatcher = containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1); @@ -1029,7 +1055,7 @@ public void search_pit_all() throws Exception { ); } - if (indexMatcher.reducedBy(user.indexMatcher("read")).isEmpty()) { + if (indexMatcher.reducedBy(user.reference(READ)).isEmpty()) { assertThat( httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") @@ -1052,7 +1078,7 @@ public void search_pit_all() throws Exception { assertThat(httpResponse, isForbidden()); } else { assertThat(httpResponse, isOk()); - assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("read"))); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.reference(READ))); } } } @@ -1063,9 +1089,9 @@ public void search_pit_static() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.post("index_a1/_search/point_in_time?keep_alive=1m"); - IndexApiResponseMatchers.OnResponseIndexMatcher indexMatcher = containsExactly(index_a1); + RestIndexMatchers.OnResponseIndexMatcher indexMatcher = containsExactly(index_a1); - if (indexMatcher.reducedBy(user.indexMatcher("read")).isEmpty()) { + if (indexMatcher.reducedBy(user.reference(READ)).isEmpty()) { assertThat( httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") @@ -1080,7 +1106,7 @@ public void search_pit_static() throws Exception { } } """, pitId)); - assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.indexMatcher("read"))); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.reference(READ))); } } } @@ -1090,7 +1116,7 @@ public void search_pit_wrongIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.post("index_a*/_search/point_in_time?keep_alive=1m"); - if (user.indexMatcher("read").coversAll(index_a1, index_a2, index_a3)) { + if (user.reference(READ).coversAll(index_a1, index_a2, index_a3)) { assertThat(httpResponse, isOk()); String pitId = httpResponse.getTextFromJsonBody("/pit_id"); httpResponse = restClient.postJson("index_b*/_search?size=1000", String.format(""" @@ -1144,7 +1170,7 @@ public void search_template_staticIndices() throws Exception { assertThat( httpResponse, - containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -1160,9 +1186,7 @@ public void msearch_staticIndices() throws Exception { """); assertThat( httpResponse, - containsExactly(index_b1, index_b2).at("responses[*].hits.hits[*]._index") - .reducedBy(user.indexMatcher("read")) - .whenEmpty(isOk()) + containsExactly(index_b1, index_b2).at("responses[*].hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isOk()) ); } } @@ -1188,7 +1212,7 @@ public void mget() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_b1, index_b2).at("docs[?(@.found == true)]._index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isOk()) ); } @@ -1212,7 +1236,7 @@ public void mget_alias() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.postJson("/_mget", mget); assertThat( httpResponse, - containsExactly(index_c1).at("docs[?(@.found == true)]._index").reducedBy(user.indexMatcher("read")).whenEmpty(isOk()) + containsExactly(index_c1).at("docs[?(@.found == true)]._index").reducedBy(user.reference(READ)).whenEmpty(isOk()) ); } } @@ -1223,7 +1247,7 @@ public void get() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("index_b1/_doc/" + testDocumentB1.id()); - assertThat(httpResponse, containsExactly(index_b1).at("_index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + assertThat(httpResponse, containsExactly(index_b1).at("_index").reducedBy(user.reference(READ)).whenEmpty(isForbidden())); } } @@ -1233,7 +1257,7 @@ public void get_alias() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("alias_c1/_doc/" + testDocumentC1.id()); - assertThat(httpResponse, containsExactly(index_c1).at("_index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + assertThat(httpResponse, containsExactly(index_c1).at("_index").reducedBy(user.reference(READ)).whenEmpty(isForbidden())); } } @@ -1275,7 +1299,7 @@ public void cat_indices_all() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("$[*].index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) ); } else { @@ -1293,7 +1317,7 @@ public void cat_indices_all() throws Exception { index_hidden_dot, index_hidden, system_index_plugin - ).at("$[*].index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + ).at("$[*].index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -1307,7 +1331,7 @@ public void cat_indices_pattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3).at("$[*].index") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -1322,7 +1346,7 @@ public void cat_indices_all_includeHidden() throws Exception { } else { assertThat( httpResponse, - containsExactly(ALL_INDICES).at("$[*].index").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(ALL_INDICES).at("$[*].index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -1336,11 +1360,11 @@ public void cat_aliases_all() throws Exception { if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { assertThat(httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias")); } else { - if (!user.indexMatcher("get_alias").isEmpty()) { + if (!user.reference(GET_ALIAS).isEmpty()) { assertThat( httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias") - .reducedBy(user.indexMatcher("get_alias")) + .reducedBy(user.reference(GET_ALIAS)) .whenEmpty(isOk()) ); } else { @@ -1355,10 +1379,10 @@ public void cat_aliases_pattern() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases/alias_a*?format=json"); - if (!user.indexMatcher("get_alias").isEmpty()) { + if (!user.reference(GET_ALIAS).isEmpty()) { assertThat( httpResponse, - containsExactly(alias_ab1).at("$[*].alias").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isOk()) + containsExactly(alias_ab1).at("$[*].alias").reducedBy(user.reference(GET_ALIAS)).whenEmpty(isOk()) ); } else { assertThat(httpResponse, isForbidden()); @@ -1375,7 +1399,7 @@ public void index_stats_all() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices.keys()") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { @@ -1394,7 +1418,7 @@ public void index_stats_all() throws Exception { index_hidden_dot, system_index_plugin ).at("indices.keys()") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -1409,7 +1433,7 @@ public void index_stats_pattern() throws Exception { assertThat( httpResponse, containsExactly(index_b1, index_b2, index_b3).at("indices.keys()") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -1427,13 +1451,13 @@ public void getAlias_all() throws Exception { assertThat( httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()") - .reducedBy(user.indexMatcher("get_alias")) + .reducedBy(user.reference(GET_ALIAS)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); assertThat( httpResponse, containsExactly(ALL_INDICES).at("$.keys()") - .reducedBy(user.indexMatcher("get_alias")) + .reducedBy(user.reference(GET_ALIAS)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -1455,11 +1479,11 @@ public void getAlias_staticAlias() throws Exception { } else { assertThat( httpResponse, - containsExactly(alias_c1).at("$.*.aliases.keys()").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isForbidden()) + containsExactly(alias_c1).at("$.*.aliases.keys()").reducedBy(user.reference(GET_ALIAS)).whenEmpty(isForbidden()) ); assertThat( httpResponse, - containsExactly(index_c1).at("$.keys()").reducedBy(user.indexMatcher("get_alias")).whenEmpty(isForbidden()) + containsExactly(index_c1).at("$.keys()").reducedBy(user.reference(GET_ALIAS)).whenEmpty(isForbidden()) ); } } @@ -1472,7 +1496,7 @@ public void getAlias_aliasPattern() throws Exception { if (user == LIMITED_USER_ALIAS_AB1 || user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { assertThat(httpResponse, isOk()); - assertThat(httpResponse, containsExactly(alias_ab1).at("$.*.aliases.keys()").reducedBy(user.indexMatcher("get_alias"))); + assertThat(httpResponse, containsExactly(alias_ab1).at("$.*.aliases.keys()").reducedBy(user.reference(GET_ALIAS))); assertThat(httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("$.keys()")); } else if (user == LIMITED_USER_ALIAS_C1 || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { // This is also a kind of anomaly in the legacy privilege evaluation: Even though we do not have permissions @@ -1529,13 +1553,13 @@ public void getAlias_indexPattern_includeHidden() throws Exception { assertThat( httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()") - .reducedBy(user.indexMatcher("get_alias")) + .reducedBy(user.reference(GET_ALIAS)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); assertThat( httpResponse, containsExactly(ALL_INDICES).at("$.keys()") - .reducedBy(user.indexMatcher("get_alias")) + .reducedBy(user.reference(GET_ALIAS)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -1568,7 +1592,7 @@ public void analyze_noIndex() throws Exception { public void analyze_staticIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.postJson("index_a1/_analyze", "{\"text\": \"sample text\"}"); - IndexMatcher matcher = containsExactly(index_a1).reducedBy(user.indexMatcher("read")); + IndexMatcher matcher = containsExactly(index_a1).reducedBy(user.reference(READ)); if (matcher.isEmpty()) { assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/analyze]")); @@ -1588,7 +1612,7 @@ public void resolve_wildcard() throws Exception { httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, alias_ab1, alias_c1).at( "$.*[*].name" - ).reducedBy(user.indexMatcher("read")).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } else { assertThat( @@ -1605,7 +1629,7 @@ public void resolve_wildcard() throws Exception { index_hidden_dot, system_index_plugin ).at("$.*[*].name") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); } @@ -1625,7 +1649,7 @@ public void resolve_wildcard_includeHidden() throws Exception { } else { assertThat( httpResponse, - containsExactly(ALL_INDICES).at("$.*[*].name").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(ALL_INDICES).at("$.*[*].name").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -1638,7 +1662,7 @@ public void resolve_indexPattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("$.*[*].name") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1653,7 +1677,7 @@ public void field_caps_all() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1662,7 +1686,7 @@ public void field_caps_all() throws Exception { assertThat( httpResponse, containsExactly(ALL_INDICES).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1678,7 +1702,7 @@ public void field_caps_indexPattern() throws Exception { assertThat( httpResponse, - containsExactly(index_b1, index_b2, index_b3).at("indices").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden()) + containsExactly(index_b1, index_b2, index_b3).at("indices").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } } @@ -1687,7 +1711,7 @@ public void field_caps_indexPattern() throws Exception { public void field_caps_staticIndices() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_field_caps?fields=*"); - assertThat(httpResponse, containsExactly(index_a1).at("indices").reducedBy(user.indexMatcher("read")).whenEmpty(isForbidden())); + assertThat(httpResponse, containsExactly(index_a1).at("indices").reducedBy(user.reference(READ)).whenEmpty(isForbidden())); } } @@ -1695,7 +1719,7 @@ public void field_caps_staticIndices() throws Exception { public void field_caps_staticIndices_hidden() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_field_caps?fields=*"); - assertThat(httpResponse, containsExactly(index_hidden).at("indices").butForbiddenIfIncomplete(user.indexMatcher("read"))); + assertThat(httpResponse, containsExactly(index_hidden).at("indices").butForbiddenIfIncomplete(user.reference(READ))); } } @@ -1707,7 +1731,7 @@ public void field_caps_alias() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(isForbidden()) ); } @@ -1721,7 +1745,7 @@ public void field_caps_aliasPattern() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1733,7 +1757,7 @@ public void field_caps_nonExisting_static() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_field_caps?fields=*"); - if (containsExactly(index_ax).reducedBy(user.indexMatcher("read")).isEmpty()) { + if (containsExactly(index_ax).reducedBy(user.reference(READ)).isEmpty()) { assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/field_caps]")); } else { assertThat(httpResponse, isNotFound()); @@ -1759,7 +1783,7 @@ public void field_caps_indexPattern_minus() throws Exception { assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") - .reducedBy(user.indexMatcher("read")) + .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -1795,7 +1819,7 @@ public void pit_delete() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.delete("_search/point_in_time", json("pit_id", List.of(indexA1pitId))); - if (user.indexMatcher("read").covers(index_a1)) { + if (user.reference(READ).covers(index_a1)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -1812,7 +1836,7 @@ public void pit_catSegments() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("_cat/pit_segments", json("pit_id", List.of(indexA1pitId))); - if (user.indexMatcher("read").covers(index_a1)) { + if (user.reference(READ).covers(index_a1)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index 5b87657ab2..ac4361089c 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -36,17 +36,17 @@ import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; -import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import org.opensearch.transport.client.Client; import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.cluster.TestRestClient.json; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimited; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; @@ -135,6 +135,41 @@ public class IndexAuthorizationReadWriteIntTests { index_hidden ); + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index write permissions of individual users. This does + * not include index creation permissions. + */ + static final TestSecurityConfig.User.MetadataKey WRITE = new TestSecurityConfig.User.MetadataKey<>( + "write", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for create index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey CREATE_INDEX = + new TestSecurityConfig.User.MetadataKey<>("create_index", RestIndexMatchers.IndexMatcher.class); + + /** + * This key identifies assertion reference data for manage index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey MANAGE_INDEX = + new TestSecurityConfig.User.MetadataKey<>("manage_index", RestIndexMatchers.IndexMatcher.class); + + /** + * This key identifies assertion reference data for alias management permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey MANAGE_ALIAS = + new TestSecurityConfig.User.MetadataKey<>("manage_alias", RestIndexMatchers.IndexMatcher.class); + // ------------------------------------------------------------------------------------------------------- // Test users with which the tests will be executed; the users need to be added to the list USERS below // The users have two redundant versions or privilege configuration, which needs to be kept in sync: @@ -155,12 +190,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("write") .on("index_aw*") )// - .indexMatcher("read", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// - .indexMatcher("write", limitedTo(index_aw1, index_aw2))// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// + .reference(WRITE, limitedTo(index_aw1, index_aw2))// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple user that can read from index_b* and write to index_bw*; the user as no privileges to create or manage indices @@ -175,12 +209,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("write") .on("index_bw*") )// - .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple user that can read from index_b* and write to index_bw*; additionally, they can create index_bw* indices @@ -197,12 +230,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("create_index") .on("index_bw*") )// - .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple user that can read from index_b* and write to index_bw*; additionally, they can create and manage index_bw* indices @@ -219,12 +251,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("manage") .on("index_bw*") )// - .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("get_alias", limitedTo()); + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2)); /** * A user that can read from index_b* and write to index_bw*; they can create and manage index_bw* indices and manage alias_bwx* aliases. @@ -245,12 +276,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("crud", "manage", "manage_aliases") .on("alias_bwx*") )// - .indexMatcher("read", limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// - .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("get_alias", limitedTo(alias_bwx)); + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// + .reference(MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx)); /** * This user differs from LIMITED_USER_B_MANAGE_INDEX_ALIAS the way that it does not give any direct @@ -269,12 +299,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("crud", "manage", "manage_aliases") .on("alias_bwx*") )// - .indexMatcher("read", limitedTo(index_br1, index_br2))// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedTo(alias_bwx))// - .indexMatcher("manage_alias", limitedTo(alias_bwx))// - .indexMatcher("get_alias", limitedTo(alias_bwx)); + .reference(READ, limitedTo(index_br1, index_br2))// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedTo(alias_bwx))// + .reference(MANAGE_ALIAS, limitedTo(alias_bwx)); /** * Same as LIMITED_USER_B_MANAGE_INDEX_ALIAS with the addition of read/write/manage privileges on index_hidden* @@ -295,8 +324,8 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("crud", "manage", "manage_aliases") .on("alias_bwx*") )// - .indexMatcher( - "read", + .reference( + READ, limitedTo( index_ar1, index_ar2, @@ -311,11 +340,10 @@ public class IndexAuthorizationReadWriteIntTests { index_hidden ) )// - .indexMatcher("write", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// - .indexMatcher("create_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// - .indexMatcher("manage_index", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// - .indexMatcher("manage_alias", limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// - .indexMatcher("get_alias", limitedTo(alias_bwx)); + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .reference(MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// + .reference(MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden)); /** * Same as LIMITED_USER_B with the addition of read/write/manage privileges for ".system_index_plugin", ".system_index_plugin_*" @@ -333,8 +361,8 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("manage", "system:admin/system_index") .on("index_bw*", ".system_index_plugin", ".system_index_plugin_*") )// - .indexMatcher( - "read", + .reference( + READ, limitedTo( index_br1, index_br2, @@ -346,23 +374,19 @@ public class IndexAuthorizationReadWriteIntTests { system_index_plugin_not_existing ) )// - .indexMatcher( - "write", + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing))// + .reference( + CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) )// - .indexMatcher( - "create_index", + .reference( + MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) )// - .indexMatcher( - "manage_index", + .reference( + MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) - )// - .indexMatcher( - "manage_alias", - limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) - )// - .indexMatcher("get_alias", limitedToNone()); + ); /** * A simple test user that has read privileges on alias_ab1r and write privileges on alias_ab1w*. The user @@ -378,16 +402,14 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get", "write") .on("alias_ab1w*") )// - .indexMatcher( - "read", + .reference( + READ, limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1, alias_ab1r, alias_ab1w, alias_ab1w_nowriteindex) )// - .indexMatcher("write", limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w, alias_ab1w_nowriteindex))// - .indexMatcher("create_index", limitedTo(index_aw1, index_aw2, index_bw1))// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1, alias_ab1r, alias_ab1w)); - + .reference(WRITE, limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w, alias_ab1w_nowriteindex))// + .reference(CREATE_INDEX, limitedTo(index_aw1, index_aw2, index_bw1))// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple test user that has read/only privileges on alias_ab1r and alias_ab1w*. However, they have write * privileges for the member index index_aw1. @@ -402,11 +424,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("read") .on("alias_ab1w") )// - .indexMatcher("read", limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w))// - .indexMatcher("write", limitedTo(index_aw1))// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone()); + .reference(READ, limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w))// + .reference(WRITE, limitedTo(index_aw1))// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple test user which has read/only privileges for "*" @@ -419,12 +441,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("read") .on("*") )// - .indexMatcher("read", unlimited())// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, unlimited())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple test user which has read/only privileges for "index_a*" @@ -437,12 +458,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("read") .on("index_a*") )// - .indexMatcher("read", limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple test user that only has index privileges for indices that are not used by this test. @@ -455,12 +475,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("crud", "indices_monitor") .on("index_does_not_exist_*") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A simple test user that has no index privileges at all. @@ -471,12 +490,11 @@ public class IndexAuthorizationReadWriteIntTests { new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("write", limitedToNone())// - .indexMatcher("create_index", limitedToNone())// - .indexMatcher("manage_index", limitedToNone())// - .indexMatcher("manage_alias", limitedToNone())// - .indexMatcher("get_alias", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); /** * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index @@ -492,12 +510,11 @@ public class IndexAuthorizationReadWriteIntTests { .indexPermissions("*") .on("*") )// - .indexMatcher("read", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("write", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("create_index", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("manage_index", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("manage_alias", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// - .indexMatcher("get_alias", limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx)); + .reference(READ, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(WRITE, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(CREATE_INDEX, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(MANAGE_INDEX, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(MANAGE_ALIAS, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx)); /** * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. @@ -506,12 +523,11 @@ public class IndexAuthorizationReadWriteIntTests { static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// .description("super unlimited (admin cert)")// .adminCertUser()// - .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("write", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("create_index", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("manage_index", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("manage_alias", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("get_alias", unlimitedIncludingOpenSearchSecurityIndex()); + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(WRITE, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(CREATE_INDEX, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(MANAGE_INDEX, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(MANAGE_ALIAS, unlimitedIncludingOpenSearchSecurityIndex()); static List USERS = ImmutableList.of( LIMITED_USER_A, @@ -570,10 +586,7 @@ public static void stopClusters() { public void putDocument() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.put("index_bw1/_doc/put_test_1", json("a", 1)); - assertThat( - httpResponse, - containsExactly(index_bw1).at("_index").reducedBy(user.indexMatcher("write")).whenEmpty(isForbidden()) - ); + assertThat(httpResponse, containsExactly(index_bw1).at("_index").reducedBy(user.reference(WRITE)).whenEmpty(isForbidden())); } finally { delete("index_bw1/_doc/put_test_1"); } @@ -583,7 +596,7 @@ public void putDocument() throws Exception { public void putDocument_systemIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.put(".system_index_plugin/_doc/put_test_1", json("a", 1)); - if (clusterConfig.systemIndexPrivilegeEnabled && user.indexMatcher("write").covers(system_index_plugin)) { + if (clusterConfig.systemIndexPrivilegeEnabled && user.reference(WRITE).covers(system_index_plugin)) { assertThat(httpResponse, isCreated()); } else if (user == SUPER_UNLIMITED_USER) { assertThat(httpResponse, isCreated()); @@ -606,10 +619,7 @@ public void deleteDocument() throws Exception { } HttpResponse httpResponse = restClient.delete("index_bw1/_doc/put_delete_test_1"); - assertThat( - httpResponse, - containsExactly(index_bw1).at("_index").reducedBy(user.indexMatcher("write")).whenEmpty(isForbidden()) - ); + assertThat(httpResponse, containsExactly(index_bw1).at("_index").reducedBy(user.reference(WRITE)).whenEmpty(isForbidden())); } finally { delete("index_bw1/_doc/put_delete_test_1"); } @@ -651,7 +661,7 @@ public void deleteByQuery_indexPattern() throws Exception { if (clusterConfig.legacyPrivilegeEvaluation) { // dnfof is not applicable to indices:data/write/delete/byquery, so we need privileges for all indices - if (user.indexMatcher("write").coversAll(index_aw1, index_aw2, index_bw1, index_bw2)) { + if (user.reference(WRITE).coversAll(index_aw1, index_aw2, index_bw1, index_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -671,7 +681,7 @@ public void deleteByQuery_indexPattern() throws Exception { @Test public void putDocument_bulk() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { - IndexApiResponseMatchers.IndexMatcher writePrivileges = user.indexMatcher("write"); + RestIndexMatchers.IndexMatcher writePrivileges = user.reference(WRITE); HttpResponse httpResponse = restClient.putJson("_bulk", """ {"index": {"_index": "index_aw1", "_id": "new_doc_aw1"}} @@ -701,7 +711,7 @@ public void putDocument_alias() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { HttpResponse httpResponse = restClient.put("alias_ab1w/_doc/put_doc_alias_test_1", json("a, 1")); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("write").coversAll(index_aw1, index_aw2, index_bw1)) { + if (user.reference(WRITE).coversAll(index_aw1, index_aw2, index_bw1)) { assertThat(httpResponse, isCreated()); } else { assertThat(httpResponse, isForbidden()); @@ -717,7 +727,7 @@ public void putDocument_alias_noWriteIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { HttpResponse httpResponse = restClient.put("alias_ab1w_nowriteindex/_doc/put_doc_alias_test_1", json("a, 1")); - if (containsExactly(alias_ab1w_nowriteindex).reducedBy(user.indexMatcher("write")).isEmpty()) { + if (containsExactly(alias_ab1w_nowriteindex).reducedBy(user.reference(WRITE)).isEmpty()) { assertThat(httpResponse, isForbidden()); } else { assertThat(httpResponse, isBadRequest()); @@ -742,7 +752,7 @@ public void putDocument_bulk_alias() throws Exception { assertThat( httpResponse, containsExactly(index_aw1).at("items[*].index[?(@.result == 'created')]._index") - .reducedBy(user.indexMatcher("write")) + .reducedBy(user.reference(WRITE)) .whenEmpty(isOk()) ); } else { @@ -759,7 +769,7 @@ public void putDocument_noExistingIndex() throws Exception { HttpResponse httpResponse = restClient.put("index_bwx1/_doc/put_doc_non_existing_index_test_1", json("a, 1")); assertThat( httpResponse, - containsExactly(index_bwx1).at("_index").reducedBy(user.indexMatcher("create_index")).whenEmpty(isForbidden()) + containsExactly(index_bwx1).at("_index").reducedBy(user.reference(CREATE_INDEX)).whenEmpty(isForbidden()) ); } finally { delete(index_bwx1); @@ -772,7 +782,7 @@ public void createIndex() throws Exception { HttpResponse httpResponse = restClient.putJson("index_bwx1", "{}"); assertThat( httpResponse, - containsExactly(index_bwx1).at("index").reducedBy(user.indexMatcher("create_index")).whenEmpty(isForbidden()) + containsExactly(index_bwx1).at("index").reducedBy(user.reference(CREATE_INDEX)).whenEmpty(isForbidden()) ); } finally { delete(index_bwx1); @@ -784,7 +794,7 @@ public void createIndex_systemIndex() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { HttpResponse httpResponse = restClient.putJson(".system_index_plugin_not_existing", "{}"); - if (user.indexMatcher("create_index").covers(system_index_plugin_not_existing)) { + if (user.reference(CREATE_INDEX).covers(system_index_plugin_not_existing)) { assertThat(httpResponse, isOk()); } else if (user == SUPER_UNLIMITED_USER || (user == UNLIMITED_USER && !clusterConfig.systemIndexPrivilegeEnabled)) { assertThat(httpResponse, isOk()); @@ -802,7 +812,7 @@ public void deleteIndex() throws Exception { createInitialTestObjects(index_bwx1); HttpResponse httpResponse = restClient.delete("index_bwx1"); - if (user.indexMatcher("manage_index").covers(index_bwx1)) { + if (user.reference(MANAGE_INDEX).covers(index_bwx1)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -819,7 +829,7 @@ public void deleteIndex_systemIndex() throws Exception { HttpResponse httpResponse = restClient.delete(".system_index_plugin_not_existing"); - if (clusterConfig.systemIndexPrivilegeEnabled && user.indexMatcher("manage_index").covers(system_index_plugin_not_existing)) { + if (clusterConfig.systemIndexPrivilegeEnabled && user.reference(MANAGE_INDEX).covers(system_index_plugin_not_existing)) { assertThat(httpResponse, isOk()); } else if (user == SUPER_UNLIMITED_USER) { assertThat(httpResponse, isOk()); @@ -842,7 +852,7 @@ public void createIndex_withAlias() throws Exception { }"""); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bwx1)) { + if (user.reference(MANAGE_ALIAS).covers(index_bwx1)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -861,7 +871,7 @@ public void deleteAlias_staticIndex() throws Exception { HttpResponse httpResponse = restClient.delete("index_bw1/_aliases/alias_bwx"); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bw1) || user.indexMatcher("manage_alias").covers(alias_bwx)) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1) || user.reference(MANAGE_ALIAS).covers(alias_bwx)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -903,7 +913,7 @@ public void aliases_createAlias() throws Exception { }"""); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bw1)) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -925,7 +935,7 @@ public void aliases_createAlias_indexPattern() throws Exception { ] }"""); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").coversAll(index_bw1, index_bw2)) { + if (user.reference(MANAGE_ALIAS).coversAll(index_bw1, index_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -949,7 +959,7 @@ public void aliases_deleteAlias_staticIndex() throws Exception { }"""); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bw1) || user.indexMatcher("manage_alias").covers(alias_bwx)) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1) || user.reference(MANAGE_ALIAS).covers(alias_bwx)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -996,7 +1006,7 @@ public void aliases_removeIndex() throws Exception { ] }"""); - if (user.indexMatcher("manage_index").covers(index_bwx1)) { + if (user.reference(MANAGE_INDEX).covers(index_bwx1)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); @@ -1014,7 +1024,7 @@ public void reindex() throws Exception { "source": { "index": "index_br1" }, "dest": { "index": "index_bwx1" } }"""); - if (containsExactly(index_bwx1).reducedBy(user.indexMatcher("create_index")).isEmpty()) { + if (containsExactly(index_bwx1).reducedBy(user.reference(CREATE_INDEX)).isEmpty()) { assertThat(httpResponse, isForbidden()); assertThat(cluster.getAdminCertRestClient().get("index_bwx1/_search"), isNotFound()); } else { @@ -1041,7 +1051,7 @@ public void cloneIndex() throws Exception { HttpResponse httpResponse = restClient.post(sourceIndex + "/_clone/" + targetIndex); assertThat( httpResponse, - containsExactly(index_bwx1).at("index").reducedBy(user.indexMatcher("manage_index")).whenEmpty(isForbidden()) + containsExactly(index_bwx1).at("index").reducedBy(user.reference(MANAGE_INDEX)).whenEmpty(isForbidden()) ); } finally { cluster.getInternalNodeClient() @@ -1061,7 +1071,7 @@ public void closeIndex() throws Exception { HttpResponse httpResponse = restClient.post("index_bw1/_close"); assertThat( httpResponse, - containsExactly(index_bw1).at("indices.keys()").reducedBy(user.indexMatcher("manage_index")).whenEmpty(isForbidden()) + containsExactly(index_bw1).at("indices.keys()").reducedBy(user.reference(MANAGE_INDEX)).whenEmpty(isForbidden()) ); } finally { cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("index_bw1")).actionGet(); @@ -1090,11 +1100,11 @@ public void closeIndex_openIndex() throws Exception { HttpResponse httpResponse = restClient.post("index_bw1/_close"); assertThat( httpResponse, - containsExactly(index_bw1).at("indices.keys()").reducedBy(user.indexMatcher("manage_index")).whenEmpty(isForbidden()) + containsExactly(index_bw1).at("indices.keys()").reducedBy(user.reference(MANAGE_INDEX)).whenEmpty(isForbidden()) ); httpResponse = restClient.post("index_bw1/_open"); - if (containsExactly(index_bw1).reducedBy(user.indexMatcher("manage_index")).isEmpty()) { + if (containsExactly(index_bw1).reducedBy(user.reference(MANAGE_INDEX)).isEmpty()) { assertThat(httpResponse, isForbidden()); } else { assertThat(httpResponse, isOk()); @@ -1117,7 +1127,7 @@ public void rollover_explicitTargetIndex() throws Exception { }"""); if (clusterConfig.legacyPrivilegeEvaluation) { - if (user.indexMatcher("manage_alias").covers(index_bw1) && user.indexMatcher("manage_index").covers(index_bw2)) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1) && user.reference(MANAGE_INDEX).covers(index_bw2)) { assertThat(httpResponse, isOk()); } else { assertThat(httpResponse, isForbidden()); diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java index ad222f71e3..d39a562d01 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java @@ -30,14 +30,15 @@ import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.cluster.TestRestClient.json; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnResponseIndexMatcher.containsExactly; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedTo; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.limitedToNone; -import static org.opensearch.test.framework.matcher.IndexApiResponseMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; import static org.opensearch.test.framework.matcher.RestMatchers.isOk; @@ -65,6 +66,23 @@ public class SnapshotAuthorizationIntTests { static final TestIndex index_bwx1 = TestIndex.name("index_bwx1").documentCount(10).seed(13).build(); // not initially created static final TestIndex index_bwx2 = TestIndex.name("index_bwx2").documentCount(10).seed(14).build(); // not initially created + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index write permissions of individual users. This does + * not include index creation permissions. + */ + static final TestSecurityConfig.User.MetadataKey WRITE = new TestSecurityConfig.User.MetadataKey<>( + "write", + RestIndexMatchers.IndexMatcher.class + ); + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// .description("index_a*")// .roles( @@ -75,8 +93,8 @@ public class SnapshotAuthorizationIntTests { .indexPermissions("write", "manage") .on("index_aw*") )// - .indexMatcher("read", limitedTo(index_a1, index_a2, index_awx1, index_awx2))// - .indexMatcher("write", limitedTo(index_awx1, index_awx2)); + .reference(READ, limitedTo(index_a1, index_a2, index_awx1, index_awx2))// + .reference(WRITE, limitedTo(index_awx1, index_awx2)); static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// .description("index_b*")// @@ -88,8 +106,8 @@ public class SnapshotAuthorizationIntTests { .indexPermissions("write", "manage") .on("index_bw*") )// - .indexMatcher("read", limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_bwx1, index_bwx2)); + .reference(READ, limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bwx1, index_bwx2)); static TestSecurityConfig.User LIMITED_USER_B_SYSTEM_INDEX = new TestSecurityConfig.User("limited_user_B_system_index")// .description("index_b*, .system_index_plugin")// @@ -106,8 +124,8 @@ public class SnapshotAuthorizationIntTests { .on(".system_index_plugin_not_existing") )// - .indexMatcher("read", limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_bwx1, index_bwx2, system_index_plugin_not_existing)); + .reference(READ, limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bwx1, index_bwx2, system_index_plugin_not_existing)); static TestSecurityConfig.User LIMITED_USER_AB = new TestSecurityConfig.User("limited_user_AB")// .description("index_a*, index_b*")// @@ -119,8 +137,8 @@ public class SnapshotAuthorizationIntTests { .indexPermissions("write", "manage") .on("index_aw*", "index_bw*") )// - .indexMatcher("read", limitedTo(index_a1, index_a2, index_awx1, index_awx2, index_b1, index_b2, index_bwx1, index_bwx2))// - .indexMatcher("write", limitedTo(index_awx1, index_awx2, index_bwx1, index_bwx2)); + .reference(READ, limitedTo(index_a1, index_a2, index_awx1, index_awx2, index_b1, index_b2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_awx1, index_awx2, index_bwx1, index_bwx2)); static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// .description("no index privileges")// @@ -128,8 +146,8 @@ public class SnapshotAuthorizationIntTests { new TestSecurityConfig.Role("r1")// .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") )// - .indexMatcher("read", limitedToNone())// - .indexMatcher("write", limitedToNone()); + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone()); static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// .description("unlimited")// @@ -140,12 +158,12 @@ public class SnapshotAuthorizationIntTests { .on("*")// )// - .indexMatcher( - "read", + .reference( + READ, limitedTo(index_a1, index_a2, index_a3, index_awx1, index_awx2, index_b1, index_b2, index_b3, index_bwx1, index_bwx2) )// - .indexMatcher( - "write", + .reference( + WRITE, limitedTo(index_a1, index_a2, index_a3, index_awx1, index_awx2, index_b1, index_b2, index_b3, index_bwx1, index_bwx2) ); @@ -156,8 +174,8 @@ public class SnapshotAuthorizationIntTests { static final TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// .description("super unlimited (admin cert)")// .adminCertUser()// - .indexMatcher("read", unlimitedIncludingOpenSearchSecurityIndex())// - .indexMatcher("write", unlimitedIncludingOpenSearchSecurityIndex()); + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(WRITE, unlimitedIncludingOpenSearchSecurityIndex()); static final List USERS = ImmutableList.of( LIMITED_USER_A, @@ -201,10 +219,7 @@ public void restore_singleIndex() throws Exception { "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true" ); - assertThat( - httpResponse, - containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) - ); + assertThat(httpResponse, containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); } finally { delete("_snapshot/test_repository/single_index_snapshot"); @@ -223,10 +238,7 @@ public void restore_singleIndex_rename1() throws Exception { json("rename_pattern", "index_(.+)x1", "rename_replacement", "index_$1x2") ); - assertThat( - httpResponse, - containsExactly(index_awx2).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) - ); + assertThat(httpResponse, containsExactly(index_awx2).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); } finally { delete("_snapshot/test_repository/single_index_snapshot"); @@ -245,10 +257,7 @@ public void restore_singleIndex_rename2() throws Exception { json("rename_pattern", "index_a(.*)", "rename_replacement", "index_b$1") ); - assertThat( - httpResponse, - containsExactly(index_bwx1).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) - ); + assertThat(httpResponse, containsExactly(index_bwx1).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); } finally { delete("_snapshot/test_repository/single_index_snapshot"); @@ -270,8 +279,7 @@ public void restore_singleIndex_renameToSystemIndex() throws Exception { if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { assertThat( httpResponse, - containsExactly(system_index_plugin_not_existing).at("snapshot.indices") - .butForbiddenIfIncomplete(user.indexMatcher("write")) + containsExactly(system_index_plugin_not_existing).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE)) ); } else { assertThat(httpResponse, isForbidden()); @@ -295,10 +303,7 @@ public void restore_singleIndexFromAllIndices() throws Exception { json("indices", "index_awx1") ); - assertThat( - httpResponse, - containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.indexMatcher("write")) - ); + assertThat(httpResponse, containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); } finally { delete("_snapshot/test_repository/all_index_snapshot"); diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java b/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java index 451c1aadf1..6cf9ecac53 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java @@ -95,6 +95,11 @@ public Builder rolloverAfter(int rolloverAfter) { return this; } + public Builder segmentCount(int segmentCount) { + testDataBuilder.segmentCount(segmentCount); + return this; + } + public TestDataStream build() { if (testData == null) { testData = testDataBuilder.get(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java index 70a4f7d102..5f85220422 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java @@ -107,7 +107,7 @@ public Builder setting(String name, int value) { } public Builder shards(int value) { - settings.put("index.number_of_shards", 5); + settings.put("index.number_of_shards", value); return this; } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 3e09475ae5..05e6ddad75 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -72,7 +72,7 @@ import org.opensearch.security.securityconf.impl.v7.RoleV7; import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; -import org.opensearch.test.framework.matcher.IndexApiResponseMatchers; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import org.opensearch.transport.client.Client; import static org.apache.http.HttpHeaders.AUTHORIZATION; @@ -459,7 +459,7 @@ public static final class User implements UserCredentialsHolder, ToXContentObjec String requestedTenant; private Map attributes = new HashMap<>(); private Map, Object> matchers = new HashMap<>(); - private Map indexMatchers = new HashMap<>(); + private Map indexMatchers = new HashMap<>(); private boolean adminCertUser = false; private Boolean hidden = null; @@ -514,25 +514,6 @@ public User attr(String key, String value) { return this; } - /** - * Associates an IndexMatcher with this test user. The IndexMatcher can be later used as a test oracle. - * See IndexAuthorizationReadOnlyIntTests for examples. - */ - public User indexMatcher(String key, IndexApiResponseMatchers.IndexMatcher indexMatcher) { - this.indexMatchers.put(key, indexMatcher); - return this; - } - - public IndexApiResponseMatchers.IndexMatcher indexMatcher(String key) { - IndexApiResponseMatchers.IndexMatcher result = this.indexMatchers.get(key); - - if (result != null) { - return result; - } else { - throw new RuntimeException("Unknown index matcher " + key + " in user " + this.name); - } - } - public User hash(String hash) { this.hash = hash; return this; @@ -586,7 +567,7 @@ public T reference(MetadataKey key) { if (result != null) { return key.type.cast(result); } else { - return null; + throw new RuntimeException("Unknown reference " + key + " in user " + this.name); } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java similarity index 99% rename from src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java rename to src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java index 84a3975af4..2245d8efcb 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/IndexApiResponseMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java @@ -50,7 +50,7 @@ *
  • The results of REST API calls can be also associated with a maximum space of indices the operation could work on. Combined with the user specific index matcher, one can determine the intersection of the allowed indices and thus the indices that are allowed in the particular case. The matchers support JSON path expressions to extract information on indices from the HTTP response bodies. See IndexAuthorizationReadOnlyIntTests for examples.
    • * */ -public class IndexApiResponseMatchers { +public class RestIndexMatchers { /** * Matchers that are directly used on HTTP responses From 0f6517b2a875995e28e5473c9b6684fc6478d3a6 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 7 Oct 2025 10:54:29 +0200 Subject: [PATCH 18/22] Moved test data classes to .data package Signed-off-by: Nils Bandener --- .../ServiceAccountAuthenticationTest.java | 2 +- .../privileges/PrivilegesEvaluatorTest.java | 2 +- .../dlsfls/FlsFmIntegrationTests.java | 4 +- ...taStreamAuthorizationReadOnlyIntTests.java | 28 ++++------ ...aStreamAuthorizationReadWriteIntTests.java | 10 ++-- .../IndexAuthorizationReadOnlyIntTests.java | 14 +++-- .../IndexAuthorizationReadWriteIntTests.java | 6 +-- .../SnapshotAuthorizationIntTests.java | 4 +- .../test/framework/TestSecurityConfig.java | 1 + .../test/framework/cluster/LocalCluster.java | 10 ++-- .../test/framework/{ => data}/TestAlias.java | 2 +- .../{ => data}/TestComponentTemplate.java | 2 +- .../test/framework/{ => data}/TestData.java | 46 +++++++++++----- .../framework/{ => data}/TestDataStream.java | 2 +- .../test/framework/{ => data}/TestIndex.java | 52 +++++++++---------- .../TestIndexOrAliasOrDatastream.java | 14 ++--- .../{ => data}/TestIndexTemplate.java | 2 +- .../framework/{ => data}/TestMapping.java | 2 +- .../matcher/RestDocumentMatchers.java | 2 +- .../framework/matcher/RestIndexMatchers.java | 4 +- 20 files changed, 112 insertions(+), 97 deletions(-) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestAlias.java (99%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestComponentTemplate.java (98%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestData.java (95%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestDataStream.java (98%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestIndex.java (81%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestIndexOrAliasOrDatastream.java (72%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestIndexTemplate.java (98%) rename src/integrationTest/java/org/opensearch/test/framework/{ => data}/TestMapping.java (96%) diff --git a/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java index e27845d95b..34857ea2a7 100644 --- a/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java @@ -20,11 +20,11 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED; import static org.opensearch.security.support.ConfigConstants.SECURITY_SYSTEM_INDICES_ENABLED_KEY; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index adc0b212f5..94a10233e0 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -19,12 +19,12 @@ import org.opensearch.script.mustache.MustacheModulePlugin; import org.opensearch.script.mustache.RenderSearchTemplateAction; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java b/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java index 068972f9e2..abee5eb844 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java @@ -28,12 +28,12 @@ import org.bouncycastle.util.encoders.Hex; import org.opensearch.plugin.mapper.MapperSizePlugin; -import org.opensearch.test.framework.TestData; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestData; +import org.opensearch.test.framework.data.TestIndex; import com.rfksystems.blake2b.Blake2b; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index 3de08d4832..74151c627b 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -21,20 +21,20 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.TestComponentTemplate; -import org.opensearch.test.framework.TestDataStream; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; -import org.opensearch.test.framework.TestIndexTemplate; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestComponentTemplate; +import org.opensearch.test.framework.data.TestDataStream; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.data.TestIndexTemplate; import org.opensearch.test.framework.matcher.RestIndexMatchers; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.data.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; @@ -523,9 +523,7 @@ public void search_termsAggregation_index() throws Exception { if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { assertThat( httpResponse, - containsExactly(ALL_INDICES).at("aggregations.indices.buckets[*].key") - .reducedBy(user.reference(READ)) - .whenEmpty(isOk()) + containsExactly(ALL_INDICES).at("aggregations.indices.buckets[*].key").reducedBy(user.reference(READ)).whenEmpty(isOk()) ); } else { // Users without full privileges will not see hidden indices here; thus on a cluster with only data streams, the result is @@ -587,8 +585,7 @@ public void getDataStream_all() throws Exception { // The legacy mode does not support dnfof for indices:admin/data_stream/get assertThat( httpResponse, - containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name") - .butForbiddenIfIncomplete(user.reference(READ)) + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -600,8 +597,7 @@ public void getDataStream_wildcard() throws Exception { // The legacy mode does not support dnfof for indices:admin/data_stream/get assertThat( httpResponse, - containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name") - .butForbiddenIfIncomplete(user.reference(READ)) + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -648,8 +644,7 @@ public void getDataStreamStats_all() throws Exception { // The legacy mode does not support dnfof for indices:monitor/data_stream/stats assertThat( httpResponse, - containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream") - .butForbiddenIfIncomplete(user.reference(READ)) + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) ); } } @@ -661,8 +656,7 @@ public void getDataStreamStats_wildcard() throws Exception { // The legacy mode does not support dnfof for indices:monitor/data_stream/stats assertThat( httpResponse, - containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream") - .butForbiddenIfIncomplete(user.reference(READ)) + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) ); } } diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java index ac6b850b8e..11a341f726 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java @@ -26,16 +26,16 @@ import org.junit.runner.RunWith; import org.opensearch.action.admin.indices.refresh.RefreshRequest; -import org.opensearch.test.framework.TestComponentTemplate; -import org.opensearch.test.framework.TestDataStream; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; -import org.opensearch.test.framework.TestIndexTemplate; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.data.TestComponentTemplate; +import org.opensearch.test.framework.data.TestDataStream; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.data.TestIndexTemplate; import org.opensearch.test.framework.matcher.RestIndexMatchers; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index f6181c475c..f96a64b1a8 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -30,21 +30,21 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.script.mustache.MustacheModulePlugin; -import org.opensearch.test.framework.TestAlias; -import org.opensearch.test.framework.TestData; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestAlias; +import org.opensearch.test.framework.data.TestData; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.matcher.RestIndexMatchers; import static java.util.stream.Collectors.joining; import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; import static org.hamcrest.MatcherAssert.assertThat; -import static org.opensearch.test.framework.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.data.TestIndex.openSearchSecurityConfigIndex; import static org.opensearch.test.framework.matcher.RestIndexMatchers.IndexMatcher; import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; @@ -697,9 +697,7 @@ public void search_staticIndices_systemIndex_alias() throws Exception { } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { assertThat( httpResponse, - containsExactly(system_index_plugin).at("hits.hits[*]._index") - .reducedBy(user.reference(READ)) - .whenEmpty(isForbidden()) + containsExactly(system_index_plugin).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) ); } else { if (user.reference(READ).covers(alias_with_system_index)) { diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java index ac4361089c..fe685bc2a0 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -28,14 +28,14 @@ import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.common.settings.Settings; -import org.opensearch.test.framework.TestAlias; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.data.TestAlias; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.matcher.RestIndexMatchers; import org.opensearch.transport.client.Client; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java index d39a562d01..f96a1ff3a8 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java @@ -25,11 +25,11 @@ import org.junit.runner.RunWith; import org.opensearch.action.admin.indices.refresh.RefreshRequest; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.matcher.RestIndexMatchers; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index 05e6ddad75..4f94af9796 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -72,6 +72,7 @@ import org.opensearch.security.securityconf.impl.v7.RoleV7; import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; +import org.opensearch.test.framework.data.TestIndex; import org.opensearch.test.framework.matcher.RestIndexMatchers; import org.opensearch.transport.client.Client; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index a11462f3a4..5aef9f29b4 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -58,17 +58,17 @@ import org.opensearch.test.framework.AuthFailureListeners; import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.OnBehalfOfConfig; -import org.opensearch.test.framework.TestAlias; -import org.opensearch.test.framework.TestComponentTemplate; -import org.opensearch.test.framework.TestDataStream; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexTemplate; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.XffConfig; import org.opensearch.test.framework.audit.TestRuleAuditLogSink; import org.opensearch.test.framework.certificate.CertificateData; import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.data.TestAlias; +import org.opensearch.test.framework.data.TestComponentTemplate; +import org.opensearch.test.framework.data.TestDataStream; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexTemplate; import org.opensearch.transport.client.Client; /** diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestAlias.java similarity index 99% rename from src/integrationTest/java/org/opensearch/test/framework/TestAlias.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestAlias.java index aff626f444..6b964b6b6b 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestAlias.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestAlias.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.util.Collections; import java.util.HashMap; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestComponentTemplate.java similarity index 98% rename from src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestComponentTemplate.java index 8d3450ab5a..9beff16754 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestComponentTemplate.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestComponentTemplate.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.util.Map; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestData.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java similarity index 95% rename from src/integrationTest/java/org/opensearch/test/framework/TestData.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestData.java index 0e949c2734..234a013ba0 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestData.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java @@ -8,7 +8,7 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.nio.ByteBuffer; import java.time.Instant; @@ -146,6 +146,8 @@ public void createIndex(Client client, String name, Settings settings) { + refreshAfter ); + Random random = new Random(subRandomSeed); + long start = System.currentTimeMillis(); String mapping = """ { "_doc": { @@ -178,18 +180,35 @@ public void createIndex(Client client, String name, Settings settings) { """; client.admin().indices().create(new CreateIndexRequest(name).settings(settings).mapping(mapping)).actionGet(); + int nextRefresh = (int) Math.floor((random.nextGaussian() * 0.5 + 0.5) * refreshAfter); + int i = 0; - this.putDocuments(client, name, -1); + for (Map.Entry entry : allDocuments.entrySet()) { + String id = entry.getKey(); + TestDocument document = entry.getValue(); + + client.index(new IndexRequest(name).source(document.content, XContentType.JSON).id(id)).actionGet(); + + if (i > nextRefresh) { + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + double g = random.nextGaussian(); + + nextRefresh = (int) Math.floor((g * 0.5 + 1) * refreshAfter) + i + 1; + } + + i++; + } + + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + + for (String id : deletedDocuments) { + client.delete(new DeleteRequest(name, id)).actionGet(); + } + + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + log.info("Test index creation finished after " + (System.currentTimeMillis() - start) + " ms"); } - /** - * Writes the documents from this TestData instance to the given index. - * - * @param client the client to be used - * @param name the name of the target index - * @param rolloverAfter if this is not -1, a rollover operation will be executed for every n documents. This is useful - * for creating several generations of data stream backing indices. - */ public void putDocuments(Client client, String name, int rolloverAfter) { try { Random random = new Random(subRandomSeed); @@ -209,12 +228,15 @@ public void putDocuments(Client client, String name, int rolloverAfter) { if (i > nextRefresh) { client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); - nextRefresh = (int) Math.floor((random.nextGaussian() * 0.5 + 1) * refreshAfter) + i + 1; + double g = random.nextGaussian(); + + nextRefresh = (int) Math.floor((g * 0.5 + 1) * refreshAfter) + i + 1; + log.debug("refresh at " + i + " " + g + " " + (g * 0.5 + 1)); } if (i > nextRollover) { - // By using rollover, we make sure that we get several generations of backing indices client.admin().indices().rolloverIndex(new RolloverRequest(name, null)); + nextRollover += rolloverAfter; } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestDataStream.java similarity index 98% rename from src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestDataStream.java index 6cf9ecac53..6d73312b5e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestDataStream.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestDataStream.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.util.Map; import java.util.Set; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndex.java similarity index 81% rename from src/integrationTest/java/org/opensearch/test/framework/TestIndex.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestIndex.java index 5f85220422..9fe0d49581 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndex.java @@ -1,32 +1,32 @@ /* -* Copyright 2021-2022 floragunn GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -*/ + * Copyright 2021-2022 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ /* -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -*/ - -package org.opensearch.test.framework; + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; import java.util.Map; import java.util.Set; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java similarity index 72% rename from src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java index f1e23f2417..48cf432c28 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndexOrAliasOrDatastream.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.util.Map; import java.util.Set; @@ -40,18 +40,18 @@ default TestIndexOrAliasOrDatastream intersection(TestIndexOrAliasOrDatastream o return this; } - static void createInitialTestObjects(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + static void createInitialTestObjects(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexLikeArray) { try (Client client = cluster.getInternalNodeClient()) { - for (TestIndexOrAliasOrDatastream testIndexOrAliasOrDatastream : testIndexOrAliasOrDatastreamArray) { - testIndexOrAliasOrDatastream.create(client); + for (TestIndexOrAliasOrDatastream testIndexLike : testIndexLikeArray) { + testIndexLike.create(client); } } } - static void delete(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + static void delete(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexLikeArray) { try (Client client = cluster.getInternalNodeClient()) { - for (TestIndexOrAliasOrDatastream testIndexOrAliasOrDatastream : testIndexOrAliasOrDatastreamArray) { - testIndexOrAliasOrDatastream.delete(client); + for (TestIndexOrAliasOrDatastream testIndexLike : testIndexLikeArray) { + testIndexLike.delete(client); } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexTemplate.java similarity index 98% rename from src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestIndexTemplate.java index 675638bb8b..bd09a1b5e0 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndexTemplate.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexTemplate.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.util.List; import java.util.Map; diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestMapping.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestMapping.java similarity index 96% rename from src/integrationTest/java/org/opensearch/test/framework/TestMapping.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestMapping.java index ba0ec95d0a..e323dc3733 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestMapping.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestMapping.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.util.Map; diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java index e00f19455e..17e5f73dbb 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java @@ -27,8 +27,8 @@ import org.hamcrest.DiagnosingMatcher; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.test.framework.TestData; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.data.TestData; /** * Matchers that can operate on responses of the OpenSearch REST APIs _search and _get; using various options like aggregations. diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java index 2245d8efcb..1fd4ff2fc7 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java @@ -29,9 +29,9 @@ import org.hamcrest.Matcher; import org.opensearch.security.DefaultObjectMapper; -import org.opensearch.test.framework.TestIndex; -import org.opensearch.test.framework.TestIndexOrAliasOrDatastream; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; From 48b65172c9b822e16b7851bebe4062df43f235c5 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Tue, 7 Oct 2025 13:42:35 +0200 Subject: [PATCH 19/22] Fixes Signed-off-by: Nils Bandener --- .../privileges/int_tests/SnapshotAuthorizationIntTests.java | 2 +- src/integrationTest/resources/log4j2-test.properties | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java index f96a1ff3a8..65ec63c271 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java @@ -43,7 +43,7 @@ import static org.opensearch.test.framework.matcher.RestMatchers.isOk; /** - * TODO requests on non master node + * TODO requests on non cm node */ @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties index 1ff778cb1c..d0bb23fa3f 100644 --- a/src/integrationTest/resources/log4j2-test.properties +++ b/src/integrationTest/resources/log4j2-test.properties @@ -48,14 +48,8 @@ logger.ldap.name=org.opensearch.security.auth.ldap.backend logger.ldap.level=TRACE logger.ldap.appenderRef.capturing.ref = logCapturingAppender -logger.privileges.name=org.opensearch.security.privileges -logger.privileges.level=DEBUG - # Logger required by test org.opensearch.security.TlsHostnameVerificationTests logger.securenetty4transport.name = org.opensearch.transport.netty4.ssl.SecureNetty4Transport logger.securenetty4transport.level = error logger.securenetty4transport.appenderRef.capturing.ref = logCapturingAppender - -logger.p.name=org.opensearch.security.privileges -logger.p.level=DEBUG From 2c321d613c97dbddbd2b41b88123441493179a95 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Wed, 8 Oct 2025 06:40:18 +0200 Subject: [PATCH 20/22] Fixes Signed-off-by: Nils Bandener --- ...taStreamAuthorizationReadOnlyIntTests.java | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java index 74151c627b..10a107d057 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -84,6 +84,16 @@ public class DataStreamAuthorizationReadOnlyIntTests { openSearchSecurityConfigIndex() ); + static final List ALL_INDICES_EXCEPT_SYSTEM_INDICES = List.of( + ds_a1, + ds_a2, + ds_a3, + ds_b1, + ds_b2, + ds_b3, + index_c1 + ); + static final List ALL_DATA_STREAMS = List.of(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3); /** @@ -236,7 +246,7 @@ public void search_noPattern() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000"); assertThat( httpResponse, - containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3, index_c1).at("hits.hits[*]._index") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -270,7 +280,7 @@ public void search_noPattern_allowNoIndicesFalse() throws Exception { assertThat( httpResponse, - containsExactly(ALL_INDICES).at("hits.hits[*]._index") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isNotFound() : isForbidden()) ); @@ -283,7 +293,7 @@ public void search_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000"); assertThat( httpResponse, - containsExactly(ALL_INDICES).at("hits.hits[*]._index") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -314,9 +324,10 @@ public void search_all_noWildcards() throws Exception { public void search_wildcard() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000"); + assertThat( httpResponse, - containsExactly(ALL_INDICES).at("hits.hits[*]._index") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -523,7 +534,9 @@ public void search_termsAggregation_index() throws Exception { if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { assertThat( httpResponse, - containsExactly(ALL_INDICES).at("aggregations.indices.buckets[*].key").reducedBy(user.reference(READ)).whenEmpty(isOk()) + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("aggregations.indices.buckets[*].key") + .reducedBy(user.reference(READ)) + .whenEmpty(isOk()) ); } else { // Users without full privileges will not see hidden indices here; thus on a cluster with only data streams, the result is @@ -558,7 +571,7 @@ public void index_stats_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_stats"); assertThat( httpResponse, - containsExactly(ALL_INDICES).at("indices.keys()") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("indices.keys()") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -690,7 +703,7 @@ public void resolve_wildcard() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*"); assertThat( httpResponse, - containsExactly(ALL_INDICES).at("$.*[*].name") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("$.*[*].name") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); @@ -716,7 +729,7 @@ public void field_caps_all() throws Exception { TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); assertThat( httpResponse, - containsExactly(ALL_INDICES).at("indices") + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("indices") .reducedBy(user.reference(READ)) .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) ); From 66f19f0dae0edcbef41b62a26136a3a5cbcd43e9 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Mon, 13 Oct 2025 09:52:09 +0200 Subject: [PATCH 21/22] Review remarks Signed-off-by: Nils Bandener --- .../IndexAuthorizationReadOnlyIntTests.java | 2 ++ .../framework/cluster/TestRestClient.java | 5 +++ .../data/TestIndexOrAliasOrDatastream.java | 12 ------- .../framework/matcher/RestIndexMatchers.java | 31 ++++++++++--------- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java index f96a64b1a8..5182f6fb53 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -885,6 +885,8 @@ public void search_alias_pattern() throws Exception { public void search_alias_pattern_negation() throws Exception { try (TestRestClient restClient = cluster.getRestClient(user)) { TestRestClient.HttpResponse httpResponse = restClient.get("alias_*,-alias_ab1/_search?size=1000"); + // Another interesting effect: The negation on alias names does actually have no effect. + // This is this time a bug in core. TODO: File issue assertThat( httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index bc279c6c95..ec0f79e73c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -525,6 +525,11 @@ public void close() { // TODO: Is there anything to clean up here? } + /** + * Helper method to create very simple dynamic JSON request bodies. + * @param attributes Key-value pairs, keys on even indices, values on odd indices. + * @return A request body that can be passed to the get(), post(), etc. methods. + */ public static HttpEntity json(Object... attributes) { Map map = new HashMap<>(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java index 48cf432c28..2e1f6ae795 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java @@ -28,18 +28,6 @@ public interface TestIndexOrAliasOrDatastream { void delete(Client client); - default TestIndexOrAliasOrDatastream intersection(TestIndexOrAliasOrDatastream other) { - if (other == this) { - return this; - } - - if (!this.name().equals(other.name())) { - throw new IllegalArgumentException("Cannot intersect different indices: " + this + " vs " + other); - } - - return this; - } - static void createInitialTestObjects(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexLikeArray) { try (Client client = cluster.getInternalNodeClient()) { for (TestIndexOrAliasOrDatastream testIndexLike : testIndexLikeArray) { diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java index 1fd4ff2fc7..ed1d9e8901 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java @@ -29,6 +29,7 @@ import org.hamcrest.Matcher; import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.cluster.TestRestClient; import org.opensearch.test.framework.data.TestIndex; import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; @@ -110,17 +111,17 @@ public static OnResponseIndexMatcher containsExactly(TestIndexOrAliasOrDatastrea public static OnResponseIndexMatcher containsExactly(Collection testIndices) { Map indexNameMap = new HashMap<>(); - boolean containsOpenSearchIndices = false; + boolean containsOpenSearchSecurityIndex = false; for (TestIndexOrAliasOrDatastream testIndex : testIndices) { if (testIndex == TestIndex.openSearchSecurityConfigIndex()) { - containsOpenSearchIndices = true; + containsOpenSearchSecurityIndex = true; } else { indexNameMap.put(testIndex.name(), testIndex); } } - return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchIndices); + return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchSecurityIndex); } } @@ -341,7 +342,7 @@ public int size() { if (!containsOpenSearchSecurityIndex) { return expectedIndices.size(); } else { - throw new RuntimeException("Size cannot be exactly specified because containsOpenSearchIndices is true"); + return expectedIndices.size() + 1; } } @@ -368,7 +369,7 @@ protected Map testIndicesIntersection( continue; } - result.put(key, index1.intersection(index2)); + result.put(key, index1); } return Collections.unmodifiableMap(result); @@ -403,17 +404,17 @@ protected static String formatResponse(TestRestClient.HttpResponse response) { static class ContainsExactlyMatcher extends AbstractIndexMatcher implements OnResponseIndexMatcher { private static final Pattern DS_BACKING_INDEX_PATTERN = Pattern.compile("\\.ds-(.+)-[0-9]+"); - ContainsExactlyMatcher(Map indexNameMap, boolean containsOpenSearchIndices) { - super(indexNameMap, containsOpenSearchIndices); + ContainsExactlyMatcher(Map indexNameMap, boolean containsOpenSearchSecurityIndex) { + super(indexNameMap, containsOpenSearchSecurityIndex); } ContainsExactlyMatcher( Map indexNameMap, - boolean containsOpenSearchIndices, + boolean containsOpenSearchSecurityIndex, String jsonPath, RestMatchers.HttpResponseMatcher statusCodeWhenEmpty ) { - super(indexNameMap, containsOpenSearchIndices, jsonPath, statusCodeWhenEmpty); + super(indexNameMap, containsOpenSearchSecurityIndex, jsonPath, statusCodeWhenEmpty); } @Override @@ -454,7 +455,7 @@ protected boolean matchesByIndices( for (Object object : collection) { String index = object.toString(); - if (containsOpenSearchSecurityIndex && (index.startsWith(".opendistro"))) { + if (containsOpenSearchSecurityIndex && (index.equals(ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX))) { seenOpenSearchIndicesBuilder.add(index); } else if (index.startsWith(".ds-")) { // We do a special treatment for data stream backing indices. We convert these to the normal data streams if expected @@ -708,14 +709,14 @@ public OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices) { */ static class UnlimitedMatcher extends DiagnosingMatcher implements OnUserIndexMatcher { - private final boolean containsOpenSearchIndices; + private final boolean containsOpenSearchSecurityIndex; UnlimitedMatcher() { - this.containsOpenSearchIndices = false; + this.containsOpenSearchSecurityIndex = false; } - UnlimitedMatcher(boolean containsOpenSearchIndices) { - this.containsOpenSearchIndices = containsOpenSearchIndices; + UnlimitedMatcher(boolean containsOpenSearchSecurityIndex) { + this.containsOpenSearchSecurityIndex = containsOpenSearchSecurityIndex; } @Override @@ -743,7 +744,7 @@ public boolean isEmpty() { @Override public boolean containsOpenSearchSecurityIndex() { - return containsOpenSearchIndices; + return containsOpenSearchSecurityIndex; } @Override From f2538e80617990fad5d1731e2acdc088be82be15 Mon Sep 17 00:00:00 2001 From: Nils Bandener Date: Fri, 17 Oct 2025 12:36:17 +0200 Subject: [PATCH 22/22] Added comment Signed-off-by: Nils Bandener --- .../java/org/opensearch/test/framework/data/TestData.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java index 234a013ba0..9f2f97496c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java @@ -235,6 +235,7 @@ public void putDocuments(Client client, String name, int rolloverAfter) { } if (i > nextRollover) { + // This creates several generations of backing indices for data streams client.admin().indices().rolloverIndex(new RolloverRequest(name, null)); nextRollover += rolloverAfter;