Add role mapping cleanup task

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -224,6 +224,7 @@ static TransportVersion def(int id) {
      * </ul>
      */
     public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION_8_15 = def(8_702_00_3);
+    public static final TransportVersion ADD_ROLE_MAPPING_CLEANUP_TASK = def(8_702_00_4);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -89,6 +89,7 @@
 import org.elasticsearch.xpack.core.security.authz.permission.RemoteClusterPermissions;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
+import org.elasticsearch.xpack.core.security.support.RoleMappingCleanupTaskParams;
 import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams;
 import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage;
 import org.elasticsearch.xpack.core.slm.SnapshotLifecycleMetadata;
@@ -304,6 +305,11 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 PersistentTaskParams.class,
                 SecurityMigrationTaskParams.TASK_NAME,
                 SecurityMigrationTaskParams::new
+            ),
+            new NamedWriteableRegistry.Entry(
+                PersistentTaskParams.class,
+                RoleMappingCleanupTaskParams.TASK_NAME,
+                RoleMappingCleanupTaskParams::new
             )
         ).filter(Objects::nonNull).toList();
     }
@@ -382,6 +388,11 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
                 new ParseField(SecurityMigrationTaskParams.TASK_NAME),
                 SecurityMigrationTaskParams::fromXContent
             ),
+            new NamedXContentRegistry.Entry(
+                PersistentTaskParams.class,
+                new ParseField(RoleMappingCleanupTaskParams.TASK_NAME),
+                RoleMappingCleanupTaskParams::fromXContent
+            ),
             new NamedXContentRegistry.Entry(
                 Metadata.Custom.class,
                 new ParseField(RoleMappingMetadata.TYPE),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java

@@ -12,7 +12,6 @@
 import org.elasticsearch.cluster.AbstractNamedDiffable;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NamedDiff;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -34,16 +33,23 @@
 
 import static org.elasticsearch.cluster.metadata.Metadata.ALL_CONTEXTS;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
 
     public static final String TYPE = "role_mappings";
+    public static int ROLE_MAPPING_METADATA_VERSION = 1;
+
+    private static final ParseField VERSION_PARSE_FIELD = new ParseField("version");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<RoleMappingMetadata, Void> PARSER = new ConstructingObjectParser<>(
         TYPE,
         // serialization tests rely on the order of the ExpressionRoleMapping
-        args -> new RoleMappingMetadata(new LinkedHashSet<>((Collection<ExpressionRoleMapping>) args[0]))
+        args -> new RoleMappingMetadata(
+            new LinkedHashSet<>((Collection<ExpressionRoleMapping>) args[0]),
+            args[1] == null ? 0 : (int) args[1]
+        )
     );
 
     static {
@@ -53,23 +59,27 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Cu
             (p, c) -> ExpressionRoleMapping.parse("name_not_available_after_deserialization", p),
             new ParseField(TYPE)
         );
+        PARSER.declareIntOrNull(optionalConstructorArg(), 0, VERSION_PARSE_FIELD);
     }
 
-    private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
+    private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of(), 0);
 
     public static RoleMappingMetadata getFromClusterState(ClusterState clusterState) {
-        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
         return clusterState.metadata().custom(RoleMappingMetadata.TYPE, RoleMappingMetadata.EMPTY);
     }
 
     private final Set<ExpressionRoleMapping> roleMappings;
 
-    public RoleMappingMetadata(Set<ExpressionRoleMapping> roleMappings) {
+    private final int version;
+
+    public RoleMappingMetadata(Set<ExpressionRoleMapping> roleMappings, int version) {
+        this.version = version;
         this.roleMappings = roleMappings;
     }
 
     public RoleMappingMetadata(StreamInput input) throws IOException {
         this.roleMappings = input.readCollectionAsSet(ExpressionRoleMapping::new);
+        this.version = input.readOptionalInt();
     }
 
     public Set<ExpressionRoleMapping> getRoleMappings() {
@@ -96,13 +106,22 @@ public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput streamInput) t
     @Override
     public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
         // role mappings are serialized without their names
-        return Iterators.concat(ChunkedToXContentHelper.startArray(TYPE), roleMappings.iterator(), ChunkedToXContentHelper.endArray());
+        return Iterators.concat(
+            ChunkedToXContentHelper.field(VERSION_PARSE_FIELD.getPreferredName(), version),
+            ChunkedToXContentHelper.startArray(TYPE),
+            roleMappings.iterator(),
+            ChunkedToXContentHelper.endArray()
+        );
     }
 
     public static RoleMappingMetadata fromXContent(XContentParser parser) throws IOException {
         return PARSER.apply(parser, null);
     }
 
+    public int getVersion() {
+        return this.version;
+    }
+
     @Override
     public String getWriteableName() {
         return TYPE;
@@ -123,17 +142,17 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         final var other = (RoleMappingMetadata) o;
-        return Objects.equals(roleMappings, other.roleMappings);
+        return version == other.version && Objects.equals(roleMappings, other.roleMappings);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(roleMappings);
+        return Objects.hash(roleMappings, version);
     }
 
     @Override
     public String toString() {
-        StringBuilder builder = new StringBuilder("RoleMapping[entries=[");
+        StringBuilder builder = new StringBuilder("RoleMapping[version=[" + version + "],entries=[");
         final Iterator<ExpressionRoleMapping> entryList = roleMappings.iterator();
         boolean firstEntry = true;
         while (entryList.hasNext()) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/RoleMappingCleanupTaskParams.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.support;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.persistent.PersistentTaskParams;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class RoleMappingCleanupTaskParams implements PersistentTaskParams {
+    public static final String TASK_NAME = "role-mapping-cleanup";
+
+    public static final ObjectParser<RoleMappingCleanupTaskParams, Void> PARSER = new ObjectParser<>(
+        TASK_NAME,
+        true,
+        RoleMappingCleanupTaskParams::new
+    );
+
+    public RoleMappingCleanupTaskParams() {}
+
+    public RoleMappingCleanupTaskParams(StreamInput in) throws IOException {}
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {}
+
+    @Override
+    public String getWriteableName() {
+        return TASK_NAME;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.ADD_ROLE_MAPPING_CLEANUP_TASK;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.endObject();
+        return builder;
+    }
+
+    public static RoleMappingCleanupTaskParams fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+}

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRoleMappingsIntegTests.java

@@ -446,7 +446,7 @@ private SignedJWT getSignedJWT(JWTClaimsSet claimsSet) throws Exception {
     }
 
     private void publishRoleMappings(Set<ExpressionRoleMapping> roleMappings) throws InterruptedException {
-        RoleMappingMetadata roleMappingMetadata = new RoleMappingMetadata(roleMappings);
+        RoleMappingMetadata roleMappingMetadata = new RoleMappingMetadata(roleMappings, 0);
         List<ClusterService> clusterServices = new ArrayList<>();
         internalCluster().getInstances(ClusterService.class).forEach(clusterServices::add);
         CountDownLatch publishedClusterState = new CountDownLatch(clusterServices.size());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -217,6 +217,7 @@
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 import org.elasticsearch.xpack.core.security.support.Automatons;
+import org.elasticsearch.xpack.core.security.support.RoleMappingCleanupTaskParams;
 import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
@@ -409,6 +410,7 @@
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.ExtensionComponents;
 import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent;
+import org.elasticsearch.xpack.security.support.RoleMappingCleanupExecutor;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.security.support.SecurityMigrationExecutor;
 import org.elasticsearch.xpack.security.support.SecurityMigrations;
@@ -629,6 +631,7 @@ public class Security extends Plugin
     private final SetOnce<SecondaryAuthActions> secondaryAuthActions = new SetOnce<>();
 
     private final SetOnce<SecurityMigrationExecutor> securityMigrationExecutor = new SetOnce<>();
+    private final SetOnce<RoleMappingCleanupExecutor> roleMappingCleanupExecutor = new SetOnce<>();
 
     // Node local retry count for migration jobs that's checked only on the master node to make sure
     // submit migration jobs doesn't get out of hand and retries forever if they fail. Reset by a
@@ -853,6 +856,15 @@ Collection<Object> createComponents(
             systemIndices.getMainIndexManager(),
             scriptService
         );
+        this.roleMappingCleanupExecutor.set(
+            new RoleMappingCleanupExecutor(
+                clusterService,
+                RoleMappingCleanupTaskParams.TASK_NAME,
+                threadPool.executor(ThreadPool.Names.MANAGEMENT),
+                nativeRoleMappingStore,
+                systemIndices.getMainIndexManager()
+            )
+        );
         final ClusterStateRoleMapper clusterStateRoleMapper = new ClusterStateRoleMapper(settings, scriptService, clusterService);
         final UserRoleMapper userRoleMapper = new CompositeRoleMapper(nativeRoleMappingStore, clusterStateRoleMapper);
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
@@ -1198,7 +1210,7 @@ Collection<Object> createComponents(
             new SecurityUsageServices(realms, allRolesStore, nativeRoleMappingStore, ipFilter.get(), profileService, apiKeyService)
         );
 
-        reservedRoleMappingAction.set(new ReservedRoleMappingAction());
+        reservedRoleMappingAction.set(new ReservedRoleMappingAction(persistentTasksService));
 
         cacheInvalidatorRegistry.validate();
 
@@ -2345,7 +2357,15 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(
         SettingsModule settingsModule,
         IndexNameExpressionResolver expressionResolver
     ) {
-        return this.securityMigrationExecutor.get() != null ? List.of(this.securityMigrationExecutor.get()) : List.of();
+        List<PersistentTasksExecutor<?>> executors = new ArrayList<>();
+        if (this.securityMigrationExecutor.get() != null) {
+            executors.add(this.securityMigrationExecutor.get());
+        }
+        if (this.roleMappingCleanupExecutor.get() != null) {
+            executors.add(this.roleMappingCleanupExecutor.get());
+        }
+
+        return executors;
     }
 
     List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers() {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/ReservedRoleMappingAction.java

@@ -7,7 +7,13 @@
 
 package org.elasticsearch.xpack.security.action.rolemapping;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.persistent.PersistentTasksService;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentParser;
@@ -16,6 +22,7 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
 import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
+import org.elasticsearch.xpack.core.security.support.RoleMappingCleanupTaskParams;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -25,6 +32,7 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.XContentHelper.mapToXContentParser;
+import static org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata.ROLE_MAPPING_METADATA_VERSION;
 
 /**
  * This Action is the reserved state save version of RestPutRoleMappingAction/RestDeleteRoleMappingAction
@@ -35,6 +43,14 @@
 public class ReservedRoleMappingAction implements ReservedClusterStateHandler<List<PutRoleMappingRequest>> {
     public static final String NAME = "role_mappings";
 
+    private final PersistentTasksService persistentTasksService;
+
+    private static final Logger logger = LogManager.getLogger(ReservedRoleMappingAction.class);
+
+    public ReservedRoleMappingAction(PersistentTasksService persistentTasksService) {
+        this.persistentTasksService = persistentTasksService;
+    }
+
     @Override
     public String name() {
         return NAME;
@@ -44,8 +60,13 @@ public String name() {
     public TransformState transform(Object source, TransformState prevState) throws Exception {
         @SuppressWarnings("unchecked")
         Set<ExpressionRoleMapping> roleMappings = validate((List<PutRoleMappingRequest>) source);
-        RoleMappingMetadata newRoleMappingMetadata = new RoleMappingMetadata(roleMappings);
-        if (newRoleMappingMetadata.equals(RoleMappingMetadata.getFromClusterState(prevState.state()))) {
+        RoleMappingMetadata newRoleMappingMetadata = new RoleMappingMetadata(roleMappings, ROLE_MAPPING_METADATA_VERSION);
+        RoleMappingMetadata currentRoleMappingMetadata = RoleMappingMetadata.getFromClusterState(prevState.state());
+
+        if (currentRoleMappingMetadata.getVersion() < ROLE_MAPPING_METADATA_VERSION) {
+            submitCleanupTask();
+        }
+        if (newRoleMappingMetadata.equals(currentRoleMappingMetadata)) {
             return prevState;
         } else {
             ClusterState newState = newRoleMappingMetadata.updateClusterState(prevState.state());
@@ -57,6 +78,25 @@ public TransformState transform(Object source, TransformState prevState) throws
         }
     }
 
+    private void submitCleanupTask() {
+        persistentTasksService.sendStartRequest(
+            RoleMappingCleanupTaskParams.TASK_NAME,
+            RoleMappingCleanupTaskParams.TASK_NAME,
+            new RoleMappingCleanupTaskParams(),
+            null,
+            ActionListener.wrap((response) -> {
+                logger.info("Role mapping cleanup task submitted");
+            }, (exception) -> {
+                // Do nothing if the task is already in progress
+                if (ExceptionsHelper.unwrapCause(exception) instanceof ResourceAlreadyExistsException) {
+                    logger.info("Cleanup task already started Already started");
+                } else {
+                    logger.warn("Role mapping cleanup task failed: " + exception);
+                }
+            })
+        );
+    }
+
     @Override
     public List<PutRoleMappingRequest> fromXContent(XContentParser parser) throws IOException {
         List<PutRoleMappingRequest> result = new ArrayList<>();