Expose cluster-state role mappings in APIs (#114951) (#115387)

This PR exposes operator-defined, cluster-state role mappings in the
[Get role mappings
API](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-role-mapping.html).


Cluster-state role mappings are returned with a reserved suffix
`-read-only-operator-mapping`, to disambiguate with native role mappings
stored in the security index. CS role mappings are also marked with a
`_read_only` metadata flag. It's possible to query a CS role mapping
using its name both with and without the suffix.  

CS role mappings can be viewed via the API, but cannot be modified. To
clarify this, the PUT and DELETE role mapping endpoints return header
warnings if native role mappings that name-clash with CS role mappings
are created, modified, or deleted. 

The PR also prevents the creation or role mappings with names ending in
`-read-only-operator-mapping` to ensure that CS role mappings and native
role mappings can always be fully disambiguated.

Finally, the PR changes how CS role mappings are persisted in
cluster-state. CS role mappings are written (and read from disk) in the
`XContent` format. This format omits the role mapping's name. This means
that if CS role mappings are ever recovered from disk (e.g., during a
master-node restart), their names are erased. To address this, this PR
changes CS role mapping serialization to persist the name of a mapping
in a reserved metadata field, and recover it from metadata during
serialization. This allows us to persist the name without BWC-breaks in
role mapping `XContent` format. It also allows us to ensure that role
mappings are re-written to cluster state in the new, name-preserving
format the first time operator file settings are processed.

Depends on: #114295
Relates: ES-9628

Author: n1v0lg

docs/changelog/114951.yaml

@@ -0,0 +1,5 @@
+pr: 114951
+summary: Expose cluster-state role mappings in APIs
+area: Authentication
+type: bug
+issues: []

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java

@@ -25,9 +25,12 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -106,6 +109,10 @@ public void testRoleMappingsAppliedOnUpgrade() throws IOException {
             );
             assertThat(roleMappings, is(not(nullValue())));
             assertThat(roleMappings.size(), equalTo(1));
+            assertThat(roleMappings, is(instanceOf(Map.class)));
+            @SuppressWarnings("unchecked")
+            Map<String, Object> roleMapping = (Map<String, Object>) roleMappings;
+            assertThat(roleMapping.keySet(), contains("everyone_kibana-read-only-operator-mapping"));
         }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java

@@ -54,6 +54,18 @@
  */
 public class ExpressionRoleMapping implements ToXContentObject, Writeable {
 
+    /**
+     * Reserved suffix for read-only operator-defined role mappings.
+     * This suffix is added to the name of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_SUFFIX = "-read-only-operator-mapping";
+    /**
+     * Reserved metadata field to mark role mappings as read-only.
+     * This field is added to the metadata of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_METADATA_FLAG = "_read_only";
     private static final ObjectParser<Builder, String> PARSER = new ObjectParser<>("role-mapping", Builder::new);
 
     /**
@@ -136,6 +148,28 @@ public ExpressionRoleMapping(StreamInput in) throws IOException {
         this.metadata = in.readGenericMap();
     }
 
+    public static boolean hasReadOnlySuffix(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX);
+    }
+
+    public static void validateNoReadOnlySuffix(String name) {
+        if (hasReadOnlySuffix(name)) {
+            throw new IllegalArgumentException(
+                "Invalid mapping name [" + name + "]. [" + READ_ONLY_ROLE_MAPPING_SUFFIX + "] is not an allowed suffix"
+            );
+        }
+    }
+
+    public static String addReadOnlySuffix(String name) {
+        return name + READ_ONLY_ROLE_MAPPING_SUFFIX;
+    }
+
+    public static String removeReadOnlySuffixIfPresent(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX)
+            ? name.substring(0, name.length() - READ_ONLY_ROLE_MAPPING_SUFFIX.length())
+            : name;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.core.security.authz;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
@@ -26,8 +28,10 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -36,7 +40,11 @@
 
 public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
 
+    private static final Logger logger = LogManager.getLogger(RoleMappingMetadata.class);
+
     public static final String TYPE = "role_mappings";
+    public static final String METADATA_NAME_FIELD = "_es_reserved_role_mapping_name";
+    public static final String FALLBACK_NAME = "name_not_available_after_deserialization";
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<RoleMappingMetadata, Void> PARSER = new ConstructingObjectParser<>(
@@ -46,12 +54,7 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Cu
     );
 
     static {
-        PARSER.declareObjectArray(
-            constructorArg(),
-            // role mapping names are lost when the role mapping metadata is serialized
-            (p, c) -> ExpressionRoleMapping.parse("name_not_available_after_deserialization", p),
-            new ParseField(TYPE)
-        );
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> parseWithNameFromMetadata(p), new ParseField(TYPE));
     }
 
     private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
@@ -153,4 +156,64 @@ public EnumSet<Metadata.XContentContext> context() {
         // are not persisted.
         return ALL_CONTEXTS;
     }
+
+    /**
+     * Ensures role mapping names are preserved when stored on disk using XContent format,
+     * which omits names. This method copies the role mapping's name into a reserved metadata field
+     * during serialization, allowing recovery during deserialization (e.g., after a master-node restart).
+     * {@link #parseWithNameFromMetadata(XContentParser)} restores the name during parsing.
+     */
+    public static ExpressionRoleMapping copyWithNameInMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = new HashMap<>(roleMapping.getMetadata());
+        // note: can't use Maps.copyWith... since these create maps that don't support `null` values in map entries
+        if (metadata.put(METADATA_NAME_FIELD, roleMapping.getName()) != null) {
+            logger.error(
+                "Metadata field [{}] is reserved and will be overwritten with an internal system value. "
+                    + "Rename this field in your role mapping configuration.",
+                METADATA_NAME_FIELD
+            );
+        }
+        return new ExpressionRoleMapping(
+            roleMapping.getName(),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            metadata,
+            roleMapping.isEnabled()
+        );
+    }
+
+    /**
+     * If a role mapping does not yet have a name persisted in metadata, it will use a constant fallback name. This method checks if a
+     * role mapping has the fallback name.
+     */
+    public static boolean hasFallbackName(ExpressionRoleMapping expressionRoleMapping) {
+        return expressionRoleMapping.getName().equals(FALLBACK_NAME);
+    }
+
+    /**
+     * Parse a role mapping from XContent, restoring the name from a reserved metadata field.
+     * Used to parse a role mapping annotated with its name in metadata via @see {@link #copyWithNameInMetadata(ExpressionRoleMapping)}.
+     */
+    public static ExpressionRoleMapping parseWithNameFromMetadata(XContentParser parser) throws IOException {
+        ExpressionRoleMapping roleMapping = ExpressionRoleMapping.parse(FALLBACK_NAME, parser);
+        return new ExpressionRoleMapping(
+            getNameFromMetadata(roleMapping),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            roleMapping.getMetadata(),
+            roleMapping.isEnabled()
+        );
+    }
+
+    private static String getNameFromMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = roleMapping.getMetadata();
+        if (metadata.containsKey(METADATA_NAME_FIELD) && metadata.get(METADATA_NAME_FIELD) instanceof String name) {
+            return name;
+        } else {
+            // This is valid the first time we recover from cluster-state: the old format metadata won't have a name stored in metadata yet
+            return FALLBACK_NAME;
+        }
+    }
 }

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityOnTrialLicenseRestTestCase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.TestSecurityClient;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -41,9 +42,7 @@
 public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase {
     private TestSecurityClient securityClient;
 
-    @ClassRule
-    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .nodes(2)
+    public static LocalClusterConfigProvider commonTrialSecurityClusterConfig = cluster -> cluster.nodes(2)
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.ml.enabled", "false")
         .setting("xpack.license.self_generated.type", "trial")
@@ -62,8 +61,10 @@ public abstract class SecurityOnTrialLicenseRestTestCase extends ESRestTestCase
         .user("admin_user", "admin-password", ROOT_USER_ROLE, true)
         .user("security_test_user", "security-test-password", "security_test_role", false)
         .user("x_pack_rest_user", "x-pack-test-password", ROOT_USER_ROLE, true)
-        .user("cat_test_user", "cat-test-password", "cat_test_role", false)
-        .build();
+        .user("cat_test_user", "cat-test-password", "cat_test_role", false);
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().apply(commonTrialSecurityClusterConfig).build();
 
     @Override
     protected String getTestRestCluster() {