Merge branch '8.x' into backport/8.x/pr-115395

Author: elasticmachine

docs/changelog/114951.yaml

@@ -0,0 +1,5 @@
+pr: 114951
+summary: Expose cluster-state role mappings in APIs
+area: Authentication
+type: bug
+issues: []

docs/changelog/115041.yaml

@@ -0,0 +1,6 @@
+pr: 115041
+summary: Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity`
+  to 100_000
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/115308.yaml

@@ -0,0 +1,6 @@
+pr: 115308
+summary: "ESQL: Disable pushdown of WHERE past STATS"
+area: ES|QL
+type: bug
+issues:
+ - 115281

muted-tests.yml

@@ -409,6 +409,9 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/rest-api/usage/line_38}
   issue: https://github.com/elastic/elasticsearch/issues/113694
+- class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT
+  method: testEveryActionIsEitherOperatorOnlyOrNonOperator
+  issue: https://github.com/elastic/elasticsearch/issues/102992
 
 # Examples:
 #

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsRoleMappingUpgradeIT.java

@@ -25,9 +25,12 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -106,6 +109,10 @@ public void testRoleMappingsAppliedOnUpgrade() throws IOException {
             );
             assertThat(roleMappings, is(not(nullValue())));
             assertThat(roleMappings.size(), equalTo(1));
+            assertThat(roleMappings, is(instanceOf(Map.class)));
+            @SuppressWarnings("unchecked")
+            Map<String, Object> roleMapping = (Map<String, Object>) roleMappings;
+            assertThat(roleMapping.keySet(), contains("everyone_kibana-read-only-operator-mapping"));
         }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java

@@ -71,7 +71,7 @@ public class StartTrainedModelDeploymentAction extends ActionType<CreateTrainedM
     public static final AllocationStatus.State DEFAULT_WAITFOR_STATE = AllocationStatus.State.STARTED;
     public static final int DEFAULT_NUM_ALLOCATIONS = 1;
     public static final int DEFAULT_NUM_THREADS = 1;
-    public static final int DEFAULT_QUEUE_CAPACITY = 1024;
+    public static final int DEFAULT_QUEUE_CAPACITY = 10_000;
     public static final Priority DEFAULT_PRIORITY = Priority.NORMAL;
 
     public StartTrainedModelDeploymentAction() {
@@ -89,7 +89,7 @@ public static class Request extends MasterNodeRequest<Request> implements ToXCon
         /**
          * If the queue is created then we can OOM when we create the queue.
          */
-        private static final int MAX_QUEUE_CAPACITY = 1_000_000;
+        private static final int MAX_QUEUE_CAPACITY = 100_000;
 
         public static final ParseField MODEL_ID = new ParseField("model_id");
         public static final ParseField DEPLOYMENT_ID = new ParseField("deployment_id");

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java

@@ -54,6 +54,18 @@
  */
 public class ExpressionRoleMapping implements ToXContentObject, Writeable {
 
+    /**
+     * Reserved suffix for read-only operator-defined role mappings.
+     * This suffix is added to the name of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_SUFFIX = "-read-only-operator-mapping";
+    /**
+     * Reserved metadata field to mark role mappings as read-only.
+     * This field is added to the metadata of all cluster-state role mappings returned via
+     * the {@code TransportGetRoleMappingsAction} action.
+     */
+    public static final String READ_ONLY_ROLE_MAPPING_METADATA_FLAG = "_read_only";
     private static final ObjectParser<Builder, String> PARSER = new ObjectParser<>("role-mapping", Builder::new);
 
     /**
@@ -136,6 +148,28 @@ public ExpressionRoleMapping(StreamInput in) throws IOException {
         this.metadata = in.readGenericMap();
     }
 
+    public static boolean hasReadOnlySuffix(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX);
+    }
+
+    public static void validateNoReadOnlySuffix(String name) {
+        if (hasReadOnlySuffix(name)) {
+            throw new IllegalArgumentException(
+                "Invalid mapping name [" + name + "]. [" + READ_ONLY_ROLE_MAPPING_SUFFIX + "] is not an allowed suffix"
+            );
+        }
+    }
+
+    public static String addReadOnlySuffix(String name) {
+        return name + READ_ONLY_ROLE_MAPPING_SUFFIX;
+    }
+
+    public static String removeReadOnlySuffixIfPresent(String name) {
+        return name.endsWith(READ_ONLY_ROLE_MAPPING_SUFFIX)
+            ? name.substring(0, name.length() - READ_ONLY_ROLE_MAPPING_SUFFIX.length())
+            : name;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java

@@ -7,6 +7,8 @@
 
 package org.elasticsearch.xpack.core.security.authz;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.cluster.AbstractNamedDiffable;
@@ -26,8 +28,10 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -36,7 +40,11 @@
 
 public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
 
+    private static final Logger logger = LogManager.getLogger(RoleMappingMetadata.class);
+
     public static final String TYPE = "role_mappings";
+    public static final String METADATA_NAME_FIELD = "_es_reserved_role_mapping_name";
+    public static final String FALLBACK_NAME = "name_not_available_after_deserialization";
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<RoleMappingMetadata, Void> PARSER = new ConstructingObjectParser<>(
@@ -46,12 +54,7 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Cu
     );
 
     static {
-        PARSER.declareObjectArray(
-            constructorArg(),
-            // role mapping names are lost when the role mapping metadata is serialized
-            (p, c) -> ExpressionRoleMapping.parse("name_not_available_after_deserialization", p),
-            new ParseField(TYPE)
-        );
+        PARSER.declareObjectArray(constructorArg(), (p, c) -> parseWithNameFromMetadata(p), new ParseField(TYPE));
     }
 
     private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
@@ -153,4 +156,64 @@ public EnumSet<Metadata.XContentContext> context() {
         // are not persisted.
         return ALL_CONTEXTS;
     }
+
+    /**
+     * Ensures role mapping names are preserved when stored on disk using XContent format,
+     * which omits names. This method copies the role mapping's name into a reserved metadata field
+     * during serialization, allowing recovery during deserialization (e.g., after a master-node restart).
+     * {@link #parseWithNameFromMetadata(XContentParser)} restores the name during parsing.
+     */
+    public static ExpressionRoleMapping copyWithNameInMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = new HashMap<>(roleMapping.getMetadata());
+        // note: can't use Maps.copyWith... since these create maps that don't support `null` values in map entries
+        if (metadata.put(METADATA_NAME_FIELD, roleMapping.getName()) != null) {
+            logger.error(
+                "Metadata field [{}] is reserved and will be overwritten with an internal system value. "
+                    + "Rename this field in your role mapping configuration.",
+                METADATA_NAME_FIELD
+            );
+        }
+        return new ExpressionRoleMapping(
+            roleMapping.getName(),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            metadata,
+            roleMapping.isEnabled()
+        );
+    }
+
+    /**
+     * If a role mapping does not yet have a name persisted in metadata, it will use a constant fallback name. This method checks if a
+     * role mapping has the fallback name.
+     */
+    public static boolean hasFallbackName(ExpressionRoleMapping expressionRoleMapping) {
+        return expressionRoleMapping.getName().equals(FALLBACK_NAME);
+    }
+
+    /**
+     * Parse a role mapping from XContent, restoring the name from a reserved metadata field.
+     * Used to parse a role mapping annotated with its name in metadata via @see {@link #copyWithNameInMetadata(ExpressionRoleMapping)}.
+     */
+    public static ExpressionRoleMapping parseWithNameFromMetadata(XContentParser parser) throws IOException {
+        ExpressionRoleMapping roleMapping = ExpressionRoleMapping.parse(FALLBACK_NAME, parser);
+        return new ExpressionRoleMapping(
+            getNameFromMetadata(roleMapping),
+            roleMapping.getExpression(),
+            roleMapping.getRoles(),
+            roleMapping.getRoleTemplates(),
+            roleMapping.getMetadata(),
+            roleMapping.isEnabled()
+        );
+    }
+
+    private static String getNameFromMetadata(ExpressionRoleMapping roleMapping) {
+        Map<String, Object> metadata = roleMapping.getMetadata();
+        if (metadata.containsKey(METADATA_NAME_FIELD) && metadata.get(METADATA_NAME_FIELD) instanceof String name) {
+            return name;
+        } else {
+            // This is valid the first time we recover from cluster-state: the old format metadata won't have a name stored in metadata yet
+            return FALLBACK_NAME;
+        }
+    }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentRequestTests.java

@@ -67,7 +67,7 @@ public static Request createRandom() {
             request.setNumberOfAllocations(randomIntBetween(1, 8));
         }
         if (randomBoolean()) {
-            request.setQueueCapacity(randomIntBetween(1, 1000000));
+            request.setQueueCapacity(randomIntBetween(1, 100_000));
         }
         if (randomBoolean()) {
             request.setPriority(randomFrom(Priority.values()).toString());
@@ -168,7 +168,7 @@ public void testValidate_GivenQueueCapacityIsNegative() {
 
     public void testValidate_GivenQueueCapacityIsAtLimit() {
         Request request = createRandom();
-        request.setQueueCapacity(1_000_000);
+        request.setQueueCapacity(100_000);
 
         ActionRequestValidationException e = request.validate();
 
@@ -177,12 +177,12 @@ public void testValidate_GivenQueueCapacityIsAtLimit() {
 
     public void testValidate_GivenQueueCapacityIsOverLimit() {
         Request request = createRandom();
-        request.setQueueCapacity(1_000_001);
+        request.setQueueCapacity(100_001);
 
         ActionRequestValidationException e = request.validate();
 
         assertThat(e, is(not(nullValue())));
-        assertThat(e.getMessage(), containsString("[queue_capacity] must be less than 1000000"));
+        assertThat(e.getMessage(), containsString("[queue_capacity] must be less than 100000"));
     }
 
     public void testValidate_GivenTimeoutIsNegative() {
@@ -234,6 +234,6 @@ public void testDefaults() {
         assertThat(request.getNumberOfAllocations(), nullValue());
         assertThat(request.computeNumberOfAllocations(), equalTo(1));
         assertThat(request.getThreadsPerAllocation(), equalTo(1));
-        assertThat(request.getQueueCapacity(), equalTo(1024));
+        assertThat(request.getQueueCapacity(), equalTo(10_000));
     }
 }

x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec

@@ -2291,6 +2291,33 @@ m:integer |a:double |x:integer
 74999     |48249.0  |0
 ;
 
+statsWithFilterOnGroups
+required_capability: fix_filter_pushdown_past_stats
+FROM employees
+| STATS v = VALUES(emp_no) by job_positions | WHERE job_positions == "Accountant" | MV_EXPAND v | SORT v
+;
+
+v:integer | job_positions:keyword
+    10001 | Accountant
+    10012 | Accountant
+    10016 | Accountant
+    10023 | Accountant
+    10025 | Accountant
+    10028 | Accountant
+    10034 | Accountant
+    10037 | Accountant
+    10044 | Accountant
+    10045 | Accountant
+    10050 | Accountant
+    10051 | Accountant
+    10066 | Accountant
+    10081 | Accountant
+    10085 | Accountant
+    10089 | Accountant
+    10092 | Accountant
+    10094 | Accountant
+;
+
 
 statsWithFiltering
 required_capability: per_agg_filtering