diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 79a00fa1293bd..3e5cf66d5a474 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -800,10 +800,7 @@ Collection createComponents( this.persistentTasksService.set(persistentTasksService); systemIndices.getMainIndexManager().addStateListener((oldState, newState) -> { - // Only consider applying migrations if it's the master node and the security index exists - if (clusterService.state().nodes().isLocalNodeElectedMaster() && newState.indexExists()) { - applyPendingSecurityMigrations(newState); - } + applyPendingSecurityMigrations(clusterService.state(), newState); }); scriptServiceReference.set(scriptService); @@ -1223,7 +1220,12 @@ Collection createComponents( return components; } - private void applyPendingSecurityMigrations(SecurityIndexManager.State newState) { + private void applyPendingSecurityMigrations(ClusterState clusterState, SecurityIndexManager.State newState) { + // Only consider applying migrations if it's the master node and the security index exists + if (clusterState.nodes().isLocalNodeElectedMaster() == false || newState.indexExists() == false) { + return; + } + // If no migrations have been applied and the security index is on the latest version (new index), all migrations can be skipped if (newState.migrationsVersion == 0 && newState.createdOnLatestVersion) { submitPersistentMigrationTask(SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey(), false); @@ -1235,7 +1237,9 @@ private void applyPendingSecurityMigrations(SecurityIndexManager.State newState) ); // Check if next migration that has not been applied is eligible to run on the current cluster - if (nextMigration == null || systemIndices.getMainIndexManager().isEligibleSecurityMigration(nextMigration.getValue()) == false) { + if (nextMigration == null + || systemIndices.getMainIndexManager().isEligibleSecurityMigration(nextMigration.getValue()) == false + || nextMigration.getValue().checkPreconditions(clusterState) == false) { // Reset retry counter if all eligible migrations have been applied successfully nodeLocalMigrationRetryCount.set(0); } else if (nodeLocalMigrationRetryCount.get() > MAX_SECURITY_MIGRATION_RETRY_COUNT) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java index 73d1a1abcdb50..601dec5ac025a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.security.action.rolemapping; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ReservedStateMetadata; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.xcontent.XContentParser; @@ -34,6 +35,8 @@ */ public class ReservedRoleMappingAction implements ReservedClusterStateHandler> { public static final String NAME = "role_mappings"; + private static final String FILE_SETTINGS_METADATA_NAMESPACE = "file_settings"; + private static final String HANDLER_ROLE_MAPPINGS_NAME = "role_mappings"; @Override public String name() { @@ -71,6 +74,11 @@ public List fromXContent(XContentParser parser) throws IO return result; } + public static Set getFileSettingsMetadataHandlerRoleMappingKeys(ClusterState clusterState) { + ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FILE_SETTINGS_METADATA_NAMESPACE); + return fileSettingsMetadata.handlers().get(HANDLER_ROLE_MAPPINGS_NAME).keys(); + } + private Set validate(List roleMappings) { var exceptions = new ArrayList(); for (var roleMapping : roleMappings) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 6d9b0ef6aeebe..fac27c9235553 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -46,6 +46,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; import org.elasticsearch.xpack.security.SecurityFeatures; import java.time.Instant; @@ -285,6 +286,7 @@ public void clusterChanged(ClusterChangedEvent event) { Tuple available = checkIndexAvailable(event.state()); final boolean indexAvailableForWrite = available.v1(); final boolean indexAvailableForSearch = available.v2(); + final RoleMappingMetadata clusterStateRoleMappingMetadata = RoleMappingMetadata.getFromClusterState(event.state()); final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(event.state()); final int migrationsVersion = getMigrationVersionFromIndexMetadata(indexMetadata); final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion = getMinSecurityIndexMappingVersion(event.state()); @@ -324,7 +326,8 @@ public void clusterChanged(ClusterChangedEvent event) { indexUUID, allSecurityFeatures.stream() .filter(feature -> featureService.clusterHasFeature(event.state(), feature)) - .collect(Collectors.toSet()) + .collect(Collectors.toSet()), + clusterStateRoleMappingMetadata ); this.state = newState; @@ -335,6 +338,10 @@ public void clusterChanged(ClusterChangedEvent event) { } } + public RoleMappingMetadata getClusterStateRoleMappingMetadata() { + return state.clusterStateRoleMappingMetadata; + } + public static int getMigrationVersionFromIndexMetadata(IndexMetadata indexMetadata) { Map customMetadata = indexMetadata == null ? null : indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY); if (customMetadata == null) { @@ -714,7 +721,8 @@ public static class State { null, null, null, - Set.of() + Set.of(), + null ); public final Instant creationTime; public final boolean isIndexUpToDate; @@ -732,6 +740,7 @@ public static class State { public final IndexMetadata.State indexState; public final String indexUUID; public final Set securityFeatures; + public final RoleMappingMetadata clusterStateRoleMappingMetadata; public State( Instant creationTime, @@ -747,7 +756,8 @@ public State( ClusterHealthStatus indexHealth, IndexMetadata.State indexState, String indexUUID, - Set securityFeatures + Set securityFeatures, + RoleMappingMetadata clusterStateRoleMappingMetadata ) { this.creationTime = creationTime; this.isIndexUpToDate = isIndexUpToDate; @@ -763,6 +773,7 @@ public State( this.indexState = indexState; this.indexUUID = indexUUID; this.securityFeatures = securityFeatures; + this.clusterStateRoleMappingMetadata = clusterStateRoleMappingMetadata; } @Override @@ -782,7 +793,8 @@ public boolean equals(Object o) { && Objects.equals(concreteIndexName, state.concreteIndexName) && indexHealth == state.indexHealth && indexState == state.indexState - && Objects.equals(securityFeatures, state.securityFeatures); + && Objects.equals(securityFeatures, state.securityFeatures) + && Objects.equals(clusterStateRoleMappingMetadata, state.clusterStateRoleMappingMetadata); } public boolean indexExists() { @@ -803,7 +815,8 @@ public int hashCode() { indexMappingVersion, concreteIndexName, indexHealth, - securityFeatures + securityFeatures, + clusterStateRoleMappingMetadata ); } @@ -840,6 +853,8 @@ public String toString() { + '\'' + ", securityFeatures=" + securityFeatures + + ", clusterStateRoleMappingMetadata=" + + clusterStateRoleMappingMetadata + '}'; } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java index 5ec76a8dc3d01..4fe2e6edf1e15 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java @@ -11,7 +11,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -20,12 +22,20 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder; +import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; +import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import static org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction.getFileSettingsMetadataHandlerRoleMappingKeys; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SecurityMainIndexMappingVersion.ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS; /** @@ -38,11 +48,22 @@ public interface SecurityMigration { /** * Method that will execute the actual migration - needs to be idempotent and non-blocking * - * @param indexManager for the security index + * @param securityIndexManager manager for the main security index * @param client the index client * @param listener listener to provide updates back to caller */ - void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener); + void migrate(SecurityIndexManager securityIndexManager, Client client, ActionListener listener); + + /** + * Check preconditions to make sure the cluster is ready for the migration + * + * @param clusterState state of the cluster + * + * @return true if preconditions are met, false otherwise + */ + default boolean checkPreconditions(ClusterState clusterState) { + return true; + } /** * Any node features that are required for this migration to run. This makes sure that all nodes in the cluster can handle any @@ -62,37 +83,33 @@ public interface SecurityMigration { } public static final Integer ROLE_METADATA_FLATTENED_MIGRATION_VERSION = 1; + public static final Integer OPERATOR_DEFINED_ROLE_MAPPINGS_CLEANUP = 2; public static final TreeMap MIGRATIONS_BY_VERSION = new TreeMap<>( Map.of(ROLE_METADATA_FLATTENED_MIGRATION_VERSION, new SecurityMigration() { private static final Logger logger = LogManager.getLogger(SecurityMigration.class); @Override - public void migrate(SecurityIndexManager indexManager, Client client, ActionListener listener) { + public void migrate(SecurityIndexManager securityIndexManager, Client client, ActionListener listener) { BoolQueryBuilder filterQuery = new BoolQueryBuilder().filter(QueryBuilders.termQuery("type", "role")) .mustNot(QueryBuilders.existsQuery("metadata_flattened")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filterQuery).size(0).trackTotalHits(true); - SearchRequest countRequest = new SearchRequest(indexManager.getConcreteIndexName()); + SearchRequest countRequest = new SearchRequest(securityIndexManager.getConcreteIndexName()); countRequest.source(searchSourceBuilder); client.search(countRequest, ActionListener.wrap(response -> { // If there are no roles, skip migration if (response.getHits().getTotalHits().value > 0) { logger.info("Preparing to migrate [" + response.getHits().getTotalHits().value + "] roles"); - updateRolesByQuery(indexManager, client, filterQuery, listener); + updateRolesByQuery(securityIndexManager.getConcreteIndexName(), client, filterQuery, listener); } else { listener.onResponse(null); } }, listener::onFailure)); } - private void updateRolesByQuery( - SecurityIndexManager indexManager, - Client client, - BoolQueryBuilder filterQuery, - ActionListener listener - ) { - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexManager.getConcreteIndexName()); + private void updateRolesByQuery(String indexName, Client client, BoolQueryBuilder filterQuery, ActionListener listener) { + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); updateByQueryRequest.setQuery(filterQuery); updateByQueryRequest.setScript( new Script( @@ -115,6 +132,89 @@ public Set nodeFeaturesRequired() { return Set.of(SecuritySystemIndices.SECURITY_ROLES_METADATA_FLATTENED); } + @Override + public int minMappingVersion() { + return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id(); + } + }, OPERATOR_DEFINED_ROLE_MAPPINGS_CLEANUP, new SecurityMigration() { + private static final Logger logger = LogManager.getLogger(SecurityMigration.class); + + @Override + public void migrate(SecurityIndexManager securityIndexManager, Client client, ActionListener listener) { + if (securityIndexManager.getClusterStateRoleMappingMetadata().getRoleMappings().isEmpty()) { + listener.onResponse(null); + } + + getNativeRoleMappingsToDelete( + client, + securityIndexManager.getClusterStateRoleMappingMetadata() + .getRoleMappings() + .stream() + .map(ExpressionRoleMapping::getName) + .toList(), + ActionListener.wrap( + roleMappingIds -> deleteNativeRoleMappings(client, roleMappingIds.iterator(), listener), + listener::onFailure + ) + ); + } + + private void getNativeRoleMappingsToDelete( + Client client, + List clusterStateRoleMappingNames, + ActionListener> listener + ) { + if (clusterStateRoleMappingNames.isEmpty()) { + listener.onResponse(List.of()); + return; + } + getNativeRoleMappingFromNames(client, listener, clusterStateRoleMappingNames.toArray(String[]::new)); + } + + private void getNativeRoleMappingFromNames(Client client, ActionListener> listener, String... names) { + new GetRoleMappingsRequestBuilder(client).names(names).execute(ActionListener.wrap(response -> { + listener.onResponse(Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList()); + }, listener::onFailure)); + } + + private void deleteNativeRoleMappings(Client client, Iterator namesIterator, ActionListener listener) { + String name = namesIterator.next(); + new DeleteRoleMappingRequestBuilder(client).name(name) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .execute(ActionListener.wrap(response -> { + if (response.isFound() == false) { + logger.warn("Expected role mapping [" + name + "] not found during role mapping clean up."); + } + if (namesIterator.hasNext()) { + deleteNativeRoleMappings(client, namesIterator, listener); + } else { + listener.onResponse(null); + } + }, listener::onFailure)); + } + + @Override + public boolean checkPreconditions(ClusterState clusterState) { + if (getFileSettingsMetadataHandlerRoleMappingKeys(clusterState).isEmpty()) { + // No operator defined role mappings, so doesn't need to be ready since cleanup is a noop + return true; + } + + // TODO Change this to check version of role mapping metadata when available. So, if version is up to date -> needs upgrade + if (RoleMappingMetadata.getFromClusterState(clusterState).getRoleMappings().stream().anyMatch((roleMapping) -> true)) { + // Version is up-to-date for role mapping so ready for cleanup + return true; + } + + // Version is not up-to-date, can't trigger cleanup + return false; + } + + @Override + public Set nodeFeaturesRequired() { + return Set.of(SecuritySystemIndices.SECURITY_ROLE_MAPPING_CLEANUP); + } + @Override public int minMappingVersion() { return ADD_REMOTE_CLUSTER_AND_DESCRIPTION_FIELDS.id(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java index 36ea14c6e101b..77c7d19e94a9b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java @@ -61,6 +61,7 @@ public class SecuritySystemIndices { public static final NodeFeature SECURITY_PROFILE_ORIGIN_FEATURE = new NodeFeature("security.security_profile_origin"); public static final NodeFeature SECURITY_MIGRATION_FRAMEWORK = new NodeFeature("security.migration_framework"); public static final NodeFeature SECURITY_ROLES_METADATA_FLATTENED = new NodeFeature("security.roles_metadata_flattened"); + public static final NodeFeature SECURITY_ROLE_MAPPING_CLEANUP = new NodeFeature("security.role_mapping_cleanup"); /** * Security managed index mappings used to be updated based on the product version. They are now updated based on per-index mappings diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index e1c3b936e5a32..86a308eaf7861 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -2522,7 +2522,8 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { indexStatus, IndexMetadata.State.OPEN, "my_uuid", - Set.of() + Set.of(), + null ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java index 2254c78a2910c..225e7e32ddc9e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmTests.java @@ -47,7 +47,8 @@ private SecurityIndexManager.State dummyState(ClusterHealthStatus indexStatus) { indexStatus, IndexMetadata.State.OPEN, "my_uuid", - Set.of() + Set.of(), + null ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 38f01d4d18bc7..c9e829a4c91b8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -419,7 +419,8 @@ private SecurityIndexManager.State indexState(boolean isUpToDate, ClusterHealthS healthStatus, IndexMetadata.State.OPEN, "my_uuid", - Set.of() + Set.of(), + null ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 9587533d87d86..03c57338b0e70 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -1706,7 +1706,8 @@ public SecurityIndexManager.State dummyIndexState(boolean isIndexUpToDate, Clust healthStatus, IndexMetadata.State.OPEN, "my_uuid", - Set.of() + Set.of(), + null ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index f91cb567ba689..de0b761f764f4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -908,7 +908,8 @@ private SecurityIndexManager.State dummyState( healthStatus, IndexMetadata.State.OPEN, "my_uuid", - Set.of() + Set.of(), + null ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java index e3b00dfbcc6b8..d2b583893dc81 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/CacheInvalidatorRegistryTests.java @@ -69,7 +69,8 @@ public void testSecurityIndexStateChangeWillInvalidateAllRegisteredInvalidators( ClusterHealthStatus.GREEN, IndexMetadata.State.OPEN, "my_uuid", - Set.of() + Set.of(), + null ); cacheInvalidatorRegistry.onSecurityIndexStateChange(previousState, currentState); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutorTests.java index 0f63e5302a5f1..d2c892f876e50 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutorTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index b9b0531fa5b68..1c10aa5e1246a 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -80,6 +80,10 @@ BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> user username: "non_operator", password: 'x-pack-test-password', role: "superuser" } + if (bwcVersion.onOrAfter('8.4.0')) { + extraConfigFile 'operator_defined_role_mappings.json', file("${project.projectDir}/src/test/resources/operator_defined_role_mappings.json") + } + user username: "test_user", password: "x-pack-test-password" extraConfigFile 'testnode.pem', file("$outputDir/testnode.pem") diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java new file mode 100644 index 0000000000000..cee73a5c2bb93 --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRoleMappingCleanupIT.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.upgrades; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY; +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.contains; + +public class SecurityIndexRoleMappingCleanupIT extends AbstractUpgradeTestCase { + public void testRoleMigration() throws Exception { + if (CLUSTER_TYPE == ClusterType.OLD) { + // Create some role mappings that clash with operator defined ones and some that do not + createNativeRoleMapping("kibana_role_mapping", Map.of("meta", "test")); + createNativeRoleMapping("no_name_conflict", Map.of("meta", "test")); + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { + waitForMigrationCompletion(adminClient()); + // Make sure all role mapping still there (since they are operator defined) + assertAllRoleMappings(client(), "no_name_conflict", "kibana_role_mapping", "fleet_role_mapping"); + // Make sure migrated mapping is only in cluster state + assertMigratedMappingNotInSecurityIndex("kibana_role_mapping"); + // Make sure not migrated mapping is still in security index + assertNotMigratedMappingInSecurityIndex("no_name_conflict"); + // Make sure we can create a conflicting role mapping + createNativeRoleMapping("kibana_role_mapping", Map.of("meta", "test")); + // TODO Delete native role mapping here and confirm file based is still here + // TODO this could look different depending on what approach we take + assertAllRoleMappings(client(), "kibana_role_mapping", "no_name_conflict", "fleet_role_mapping"); + } + } + + @SuppressWarnings("unchecked") + private void assertMigratedMappingNotInSecurityIndex(String mappingName) throws IOException { + final Request request = new Request("POST", "/.security/_search"); + RequestOptions.Builder options = request.getOptions().toBuilder(); + request.setJsonEntity(String.format(Locale.ROOT, """ + {"query":{"bool":{"must":[{"term":{"_id":"%s_%s"}}]}}}""", "role-mapping", mappingName)); + addExpectWarningOption(options); + request.setOptions(options); + + Response response = adminClient().performRequest(request); + assertOK(response); + final Map responseMap = responseAsMap(response); + + Map hits = ((Map) responseMap.get("hits")); + assertEquals(0, ((List) hits.get("hits")).size()); + } + + @SuppressWarnings("unchecked") + private void assertNotMigratedMappingInSecurityIndex(String mappingName) throws IOException { + final Request request = new Request("POST", "/.security/_search"); + RequestOptions.Builder options = request.getOptions().toBuilder(); + request.setJsonEntity(String.format(Locale.ROOT, """ + {"query":{"bool":{"must":[{"term":{"_id":"%s_%s"}}]}}}""", "role-mapping", mappingName)); + addExpectWarningOption(options); + request.setOptions(options); + + Response response = adminClient().performRequest(request); + assertOK(response); + final Map responseMap = responseAsMap(response); + + Map hits = ((Map) responseMap.get("hits")); + assertEquals(1, ((List) hits.get("hits")).size()); + } + + private void addExpectWarningOption(RequestOptions.Builder options) { + Set expectedWarnings = Set.of( + "this request accesses system indices: [.security-7]," + + " but in a future major version, direct access to system indices will be prevented by default" + ); + + options.setWarningsHandler(warnings -> { + final Set actual = Set.copyOf(warnings); + // Return true if the warnings aren't what we expected; the client will treat them as a fatal error. + return actual.equals(expectedWarnings) == false; + }); + } + + @SuppressWarnings("unchecked") + private static void waitForMigrationCompletion(RestClient adminClient) throws Exception { + final Request request = new Request("GET", "_cluster/state/metadata/" + INTERNAL_SECURITY_MAIN_INDEX_7); + assertBusy(() -> { + Response response = adminClient.performRequest(request); + assertOK(response); + Map responseMap = responseAsMap(response); + Map indicesMetadataMap = (Map) ((Map) responseMap.get("metadata")).get( + "indices" + ); + assertTrue(indicesMetadataMap.containsKey(INTERNAL_SECURITY_MAIN_INDEX_7)); + assertTrue( + ((Map) indicesMetadataMap.get(INTERNAL_SECURITY_MAIN_INDEX_7)).containsKey(MIGRATION_VERSION_CUSTOM_KEY) + ); + }); + } + + private void createNativeRoleMapping(String roleMappingName, Map metadata) throws IOException { + final Request request = new Request("POST", "/_security/role_mapping/" + roleMappingName); + BytesReference source = BytesReference.bytes( + jsonBuilder().map( + Map.of( + ExpressionRoleMapping.Fields.ROLES.getPreferredName(), + List.of("superuser"), + ExpressionRoleMapping.Fields.ENABLED.getPreferredName(), + true, + ExpressionRoleMapping.Fields.RULES.getPreferredName(), + Map.of("field", Map.of("username", "role-mapping-test-user")), + RoleDescriptor.Fields.METADATA.getPreferredName(), + metadata + ) + ) + ); + request.setJsonEntity(source.utf8ToString()); + assertOK(client().performRequest(request)); + } + + private void assertAllRoleMappings(RestClient client, String... roleNames) throws IOException { + Request request = new Request("GET", "/_security/role_mapping"); + Response response = client.performRequest(request); + assertOK(response); + Map responseMap = responseAsMap(response); + + for (String roleName : roleNames) { + assertThat(responseMap.keySet(), contains(roleName)); + } + + assertThat(responseMap.get("count"), is(roleNames.length)); + } +} diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json b/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json new file mode 100644 index 0000000000000..f711f5921606e --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/operator_defined_role_mappings.json @@ -0,0 +1,32 @@ +{ + "metadata": { + "version": "2", + "compatibility": "8.4.0" + }, + "state": { + "role_mappings": { + "kibana_role_mapping": { + "enabled": true, + "roles": [ + "kibana_user" + ], + "rules": { + "field": { + "username": "role-mapping-test-user" + } + } + }, + "fleet_role_mapping": { + "enabled": true, + "roles": [ + "fleet_user" + ], + "rules": { + "field": { + "username": "role-mapping-test-user" + } + } + } + } + } +}