Merge remote-tracking branch 'upstream/main' into security-stats

* upstream/main:
  Turn NumericValues into functional interface (elastic#135068)
  Improve block loader for source only runtime fields of type keyword (elastic#135026)
  Mute org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT test {csv-spec:stats.StdDeviationGroupedAllTypes} elastic#135103
  Mute org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT test {csv-spec:stats.StdDeviationWithLongs} elastic#135102
  Mute org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT test {csv-spec:inlinestats.StdDevFilter} elastic#135101
  Mute org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT test {csv-spec:stats.StdDevFilter} elastic#135100
  Remove track_live_docs_in_memory_bytes feature flag (elastic#134900)
  Create SPI to allow prohibiting certain top-level mappings (elastic#132360)
  Only validate primary ids on release branches (elastic#135044)
  Added no-op support for project_routing query param to REST endpoints that will support cross-project search (elastic#134741)
  Fix race in FileSettingsServiceIT.testSettingsAppliedOnStart (elastic#134368)

Author: szybia

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -125,6 +125,9 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             tasks.named('generateTransportVersion') {
                 currentUpperBoundName = '9.2'
             }
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.2'
+            }
         """
         referableAndReferencedTransportVersion("existing_91", "8012000")
         referableAndReferencedTransportVersion("older_92", "8122000")

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -277,4 +277,21 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
             "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv]"
         )
     }
+
+    def "primary id checks skipped on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.1'
+            }
+        """
+        referableAndReferencedTransportVersion("some_tv", "8125000")
+        transportVersionUpperBound("9.2", "some_tv", "8125000")
+
+        when:
+        def result = gradleRunner("validateTransportVersionResources").build()
+
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin;
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
+import org.gradle.api.Action;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
@@ -23,7 +24,6 @@
 
 import java.util.Map;
 import java.util.Properties;
-import java.util.function.Consumer;
 
 public class TransportVersionResourcesPlugin implements Plugin<Project> {
 
@@ -70,6 +70,7 @@ public void apply(Project project) {
                 t.getReferencesFiles().setFrom(tvReferencesConfig);
                 t.getShouldValidateDensity().convention(true);
                 t.getShouldValidatePrimaryIdNotPatch().convention(true);
+                t.getCurrentUpperBoundName().convention(currentVersion.getMajor() + "." + currentVersion.getMinor());
             });
         project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
 
@@ -83,7 +84,7 @@ public void apply(Project project) {
             t.into(resourceRoot + "/definitions", c -> c.from(generateManifestTask));
         });
 
-        Consumer<GenerateTransportVersionDefinitionTask> generationConfiguration = t -> {
+        Action<GenerateTransportVersionDefinitionTask> generationConfiguration = t -> {
             t.setGroup(taskGroup);
             t.getReferencesFiles().setFrom(tvReferencesConfig);
             t.getIncrement().convention(1000);
@@ -92,17 +93,17 @@ public void apply(Project project) {
 
         var generateDefinitionsTask = project.getTasks()
             .register("generateTransportVersion", GenerateTransportVersionDefinitionTask.class, t -> {
-                generationConfiguration.accept(t);
                 t.setDescription("(Re)generates a transport version definition file");
             });
+        generateDefinitionsTask.configure(generationConfiguration);
         validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask));
 
         var resolveConflictTask = project.getTasks()
             .register("resolveTransportVersionConflict", GenerateTransportVersionDefinitionTask.class, t -> {
-                generationConfiguration.accept(t);
                 t.setDescription("Resolve merge conflicts in transport version internal state files");
                 t.getResolveConflict().set(true);
             });
+        resolveConflictTask.configure(generationConfiguration);
         validateTask.configure(t -> t.mustRunAfter(resolveConflictTask));
 
         var generateInitialTask = project.getTasks()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -60,6 +60,13 @@ public Path getResourcesDir() {
     @Input
     public abstract Property<Boolean> getShouldValidatePrimaryIdNotPatch();
 
+    /**
+     * The name of the upper bounds file which will be used at runtime on the current branch. Normally
+     * this equates to VersionProperties.getElasticsearchVersion().
+     */
+    @Input
+    public abstract Property<String> getCurrentUpperBoundName();
+
     private record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definition) {}
 
     private static final Pattern NAME_FORMAT = Pattern.compile("[a-z0-9_]+");
@@ -76,6 +83,7 @@ public void validateTransportVersions() throws IOException {
         Map<String, TransportVersionDefinition> allDefinitions = collectAllDefinitions(referableDefinitions, unreferableDefinitions);
         Map<Integer, List<IdAndDefinition>> idsByBase = collectIdsByBase(allDefinitions.values());
         Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
+        boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
 
         for (var definition : referableDefinitions.values()) {
             validateNamedDefinition(definition, referencedNames);
@@ -93,7 +101,9 @@ public void validateTransportVersions() throws IOException {
             validateUpperBound(upperBound, allDefinitions, idsByBase);
         }
 
-        validatePrimaryIds(resources, upperBounds, allDefinitions);
+        if (onReleaseBranch == false) {
+            validatePrimaryIds(resources, upperBounds, allDefinitions);
+        }
     }
 
     private Map<String, TransportVersionDefinition> collectAllDefinitions(
@@ -318,6 +328,15 @@ private void validatePrimaryIds(
         );
     }
 
+    private boolean checkIfDefinitelyOnReleaseBranch(Map<String, TransportVersionUpperBound> upperBounds) {
+        // only want to look at definitions <= the current upper bound.
+        // TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
+        String currentUpperBoundName = getCurrentUpperBoundName().get();
+        TransportVersionUpperBound currentUpperBound = upperBounds.get(currentUpperBoundName);
+
+        return upperBounds.values().stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
+    }
+
     private void throwDefinitionFailure(TransportVersionDefinition definition, String message) {
         Path relativePath = getResources().get().getDefinitionPath(definition);
         throw new VerificationException("Transport version definition file [" + relativePath + "] " + message);

docs/changelog/135026.yaml

@@ -0,0 +1,5 @@
+pr: 135026
+summary: Improve block loader for source only runtime fields of type keyword
+area: Mapping
+type: enhancement
+issues: []

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java

@@ -67,7 +67,7 @@ public List<RestHandler> getRestHandlers(
         Predicate<NodeFeature> clusterSupportsFeature
     ) {
         return Arrays.asList(
-            new RestSearchTemplateAction(clusterSupportsFeature),
+            new RestSearchTemplateAction(clusterSupportsFeature, settings),
             new RestMultiSearchTemplateAction(settings),
             new RestRenderSearchTemplateAction()
         );

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java

@@ -34,9 +34,11 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
     private static final Set<String> RESPONSE_PARAMS = Set.of(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM);
 
     private final boolean allowExplicitIndex;
+    private final Settings settings;
 
     public RestMultiSearchTemplateAction(Settings settings) {
         this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
+        this.settings = settings;
     }
 
     @Override
@@ -63,7 +65,12 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
     /**
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest}
      */
-    public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
+    public MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
+        if (settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false)) {
+            // accept but drop project_routing param until fully supported
+            restRequest.param("project_routing");
+        }
+
         MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
         if (restRequest.hasParam("max_concurrent_searches")) {
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -36,9 +37,11 @@ public class RestSearchTemplateAction extends BaseRestHandler {
     private static final Set<String> RESPONSE_PARAMS = Set.of(TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM);
 
     private final Predicate<NodeFeature> clusterSupportsFeature;
+    private final Settings settings;
 
-    public RestSearchTemplateAction(Predicate<NodeFeature> clusterSupportsFeature) {
+    public RestSearchTemplateAction(Predicate<NodeFeature> clusterSupportsFeature, Settings settings) {
         this.clusterSupportsFeature = clusterSupportsFeature;
+        this.settings = settings;
     }
 
     @Override
@@ -58,6 +61,11 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        if (settings != null && settings.getAsBoolean("serverless.cross_project.enabled", false)) {
+            // accept but drop project_routing param until fully supported
+            request.param("project_routing");
+        }
+
         // Creates the search request with all required params
         SearchRequest searchRequest = new SearchRequest();
         RestSearchAction.parseSearchRequest(

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -35,7 +36,7 @@ public void testParseRequest() throws Exception {
         RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(data), XContentType.JSON)
             .build();
 
-        MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
+        MultiSearchTemplateRequest request = new RestMultiSearchTemplateAction(Settings.EMPTY).parseRequest(restRequest, true);
 
         assertThat(request.requests().size(), equalTo(3));
         assertThat(request.requests().get(0).getRequest().indices()[0], equalTo("test0"));
@@ -73,7 +74,7 @@ public void testParseWithCarriageReturn() throws Exception {
         RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(content), XContentType.JSON)
             .build();
 
-        MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
+        MultiSearchTemplateRequest request = new RestMultiSearchTemplateAction(Settings.EMPTY).parseRequest(restRequest, true);
 
         assertThat(request.requests().size(), equalTo(1));
         assertThat(request.requests().get(0).getRequest().indices()[0], equalTo("test0"));
@@ -125,7 +126,7 @@ public void testMultiSearchTemplateToJson() throws Exception {
         // Deserialize the request
         RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent(new BytesArray(serialized), XContentType.JSON)
             .build();
-        MultiSearchTemplateRequest deser = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
+        MultiSearchTemplateRequest deser = new RestMultiSearchTemplateAction(Settings.EMPTY).parseRequest(restRequest, true);
 
         // For object equality purposes need to set the search requests' source to non-null
         for (SearchTemplateRequest str : deser.requests()) {

muted-tests.yml

@@ -411,9 +411,6 @@ tests:
 - class: org.elasticsearch.gradle.internal.transport.TransportVersionManagementPluginFuncTest
   method: cannot change committed ids to a branch
   issue: https://github.com/elastic/elasticsearch/issues/132790
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT
-  method: testSettingsAppliedOnStart
-  issue: https://github.com/elastic/elasticsearch/issues/131210
 - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests
   method: test40VerifyAutogeneratedCredentials
   issue: https://github.com/elastic/elasticsearch/issues/132877
@@ -579,6 +576,18 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeMetricsIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/135055
+- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
+  method: test {csv-spec:stats.StdDevFilter}
+  issue: https://github.com/elastic/elasticsearch/issues/135100
+- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
+  method: test {csv-spec:inlinestats.StdDevFilter}
+  issue: https://github.com/elastic/elasticsearch/issues/135101
+- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
+  method: test {csv-spec:stats.StdDeviationWithLongs}
+  issue: https://github.com/elastic/elasticsearch/issues/135102
+- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
+  method: test {csv-spec:stats.StdDeviationGroupedAllTypes}
+  issue: https://github.com/elastic/elasticsearch/issues/135103
 
 # Examples:
 #