Merge branch 'main' into feature/esql-match-options

# Conflicts:
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java

Author: carlosdelest

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -78,7 +78,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        PluginsLoader.createPluginsLoader(Set.of(), PluginsLoader.loadPluginsBundles(Path.of(System.getProperty("plugins.dir"))))
+        PluginsLoader.createPluginsLoader(Set.of(), PluginsLoader.loadPluginsBundles(Path.of(System.getProperty("plugins.dir"))), Map.of())
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -69,7 +70,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 // Pass through distribution type
                 "-Des.distribution.type=" + distroType
             ),
-            maybeEnableNativeAccess(),
+            maybeEnableNativeAccess(useEntitlements),
             maybeOverrideDockerCgroup(distroType),
             maybeSetActiveProcessorCount(nodeSettings),
             maybeSetReplayFile(distroType, isHotspot),
@@ -124,11 +125,18 @@ private static Stream<String> maybeSetActiveProcessorCount(Settings nodeSettings
         return Stream.empty();
     }
 
-    private static Stream<String> maybeEnableNativeAccess() {
+    private static Stream<String> maybeEnableNativeAccess(boolean useEntitlements) {
+        var enableNativeAccessOptions = new ArrayList<String>();
         if (Runtime.version().feature() >= 21) {
-            return Stream.of("--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core");
+            enableNativeAccessOptions.add("--enable-native-access=org.elasticsearch.nativeaccess,org.apache.lucene.core");
+            if (useEntitlements) {
+                enableNativeAccessOptions.add("--enable-native-access=ALL-UNNAMED");
+                if (Runtime.version().feature() >= 24) {
+                    enableNativeAccessOptions.add("--illegal-native-access=deny");
+                }
+            }
         }
-        return Stream.empty();
+        return enableNativeAccessOptions.stream();
     }
 
     /*

docs/changelog/119072.yaml

@@ -0,0 +1,12 @@
+pr: 119072
+summary: Turn `_source` meta fieldmapper's mode attribute into a no-op
+area: Mapping
+type: breaking
+issues:
+ - 118596
+breaking:
+  title: Turn `_source` meta fieldmapper's mode attribute into a no-op
+  area: Mapping
+  details: The `mode` mapping attribute of `_source` metadata field mapper has been turned into a no-op. Instead the `index.mapping.source.mode` index setting should be used to configure source mode.
+  impact: Configuring the `mode` attribute for the `_source` meta field mapper will have no effect on indices created with Elasticsearch 9.0.0 or later. Note that `_source.mode` configured on indices before upgrading to 9.0.0 or later will remain efficive after upgrading.
+  notable: false

docs/changelog/120494.yaml

@@ -0,0 +1,5 @@
+pr: 120494
+summary: Update grammar to rely on `indexPattern` instead of identifier in join target
+area: ES|QL
+type: enhancement
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -182,13 +182,6 @@ public void checkChangeNetworkHandling(Class<?> callerClass) {
         checkChangeJVMGlobalState(callerClass);
     }
 
-    /**
-     * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions
-     */
-    public void checkReadSensitiveNetworkInformation(Class<?> callerClass) {
-        neverEntitled(callerClass, "access sensitive network information");
-    }
-
     /**
      * Check for operations that can access sensitive network information, e.g. secrets, tokens or SSL sessions
      */

libs/native/src/main/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.nativeaccess;
+
+public class NativeAccessUtil {
+    /**
+     * Enables native access for the provided module. No-op for JDK 21 or before.
+     */
+    public static void enableNativeAccess(ModuleLayer.Controller controller, Module module) {}
+
+    public static boolean isNativeAccessEnabled(Module module) {
+        return true;
+    }
+}

libs/native/src/main22/java/org/elasticsearch/nativeaccess/NativeAccessUtil.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.nativeaccess;
+
+public class NativeAccessUtil {
+    /**
+     * Enables native access for the provided module. Available to JDK 22+, required for JDK 24+ when using --illegal-native-access=deny
+     */
+    public static void enableNativeAccess(ModuleLayer.Controller controller, Module module) {
+        controller.enableNativeAccess(module);
+    }
+
+    public static boolean isNativeAccessEnabled(Module module) {
+        return module.isNativeAccessEnabled();
+    }
+}

muted-tests.yml

@@ -241,6 +241,11 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
   method: testStopQuery
   issue: https://github.com/elastic/elasticsearch/issues/120767
+- class: org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT
+  issue: https://github.com/elastic/elasticsearch/issues/120772
+- class: org.elasticsearch.action.search.SearchProgressActionListenerIT
+  method: testSearchProgressWithHits
+  issue: https://github.com/elastic/elasticsearch/issues/120671
 
 # Examples:
 #

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SourceModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
+
+    public SourceModeRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    public void testConfigureStoredSourceBeforeIndexCreationLegacy() throws IOException {
+        assumeTrue("testing deprecation warnings and deprecation migrations", getOldClusterTestVersion().before("9.0.0"));
+        String templateName = "logs@custom";
+        if (isOldCluster()) {
+            var storedSourceMapping = """
+                {
+                  "template": {
+                    "settings": {
+                      "index": {
+                        "mode": "logsdb"
+                      }
+                    },
+                    "mappings": {
+                      "_source": {
+                        "mode": "stored"
+                      }
+                    }
+                  }
+                }""";
+            var putComponentTemplateRequest = new Request("PUT", "/_component_template/" + templateName);
+            putComponentTemplateRequest.setOptions(expectWarnings(SourceFieldMapper.DEPRECATION_WARNING));
+            putComponentTemplateRequest.setJsonEntity(storedSourceMapping);
+            assertOK(client().performRequest(putComponentTemplateRequest));
+
+            var request = new Request("GET", "/_migration/deprecations");
+            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
+            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
+            assertThat(
+                (String) nodeSettings.get("details"),
+                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
+            );
+        } else if (isUpgradedCluster()) {
+            var request = new Request("GET", "/_migration/deprecations");
+            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
+            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
+            assertThat(
+                (String) nodeSettings.get("details"),
+                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
+            );
+        }
+    }
+
+    public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOException {
+        assumeTrue("testing deprecation warnings and deprecation migrations", getOldClusterTestVersion().before("9.0.0"));
+        String templateName = "logs@custom";
+        if (isOldCluster()) {
+            var storedSourceMapping = """
+                {
+                  "template": {
+                    "mappings": {
+                      "_source": {
+                        "mode": "stored"
+                      }
+                    }
+                  }
+                }""";
+            var putComponentTemplateRequest = new Request("PUT", "/_component_template/" + templateName);
+            putComponentTemplateRequest.setOptions(expectWarnings(SourceFieldMapper.DEPRECATION_WARNING));
+            putComponentTemplateRequest.setJsonEntity(storedSourceMapping);
+            assertOK(client().performRequest(putComponentTemplateRequest));
+
+            var request = new Request("GET", "/_migration/deprecations");
+            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
+            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
+            assertThat(
+                (String) nodeSettings.get("details"),
+                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
+            );
+        } else if (isUpgradedCluster()) {
+            var request = new Request("GET", "/_migration/deprecations");
+            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
+            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
+            assertThat(
+                (String) nodeSettings.get("details"),
+                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
+            );
+        }
+    }
+}

rest-api-spec/build.gradle

@@ -87,4 +87,15 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed")
   task.skipTest("indices.create/10_basic/Create lookup index", "default auto_expand_replicas was removed")
   task.skipTest("indices.create/10_basic/Create lookup index with one shard", "default auto_expand_replicas was removed")
+  task.skipTest("range/20_synthetic_source/Double range", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("range/20_synthetic_source/Float range", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("range/20_synthetic_source/Integer range", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("range/20_synthetic_source/IP range", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("range/20_synthetic_source/Long range", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("range/20_synthetic_source/Date range Rounding Fixes", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("index/92_metrics_auto_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("logsdb/10_settings/routing path allowed in logs mode with routing on sort fields", "Unknown feature routing.logsb_route_on_sort_fields")
 })