Merge branch 'main' into feature/esql-match-options

# Conflicts:
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

Author: carlosdelest

docs/changelog/120392.yaml

@@ -0,0 +1,6 @@
+pr: 120392
+summary: Test/107515 restore template with match only text mapper it fail
+area: Search
+type: bug
+issues:
+ - 107515

docs/changelog/120617.yaml

@@ -0,0 +1,5 @@
+pr: 120617
+summary: Fix queries with document level security on lookup indexes
+area: ES|QL
+type: bug
+issues: [120509]

docs/changelog/120717.yaml

@@ -0,0 +1,6 @@
+pr: 120717
+summary: Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled
+area: Ranking
+type: bug
+issues:
+ - 120739

docs/changelog/120748.yaml

@@ -0,0 +1,15 @@
+pr: 120748
+summary: Removing support for types field in watcher search
+area: Watcher
+type: breaking
+issues: []
+breaking:
+  title: Removing support for types field in watcher search
+  area: REST API
+  details: >-
+    Previously, setting the `input.search.request.types` field in the payload when creating a watcher to an empty array
+    was  allowed, although it resulted in a deprecation warning and had no effect (and any value other than an empty
+    array would result in an error). Now, support for this field is entirely removed, and the empty array will also
+    result in an error.
+  impact: Users should stop setting this field (which did not have any effect anyway).
+  notable: false

docs/changelog/120752.yaml

@@ -0,0 +1,6 @@
+pr: 120752
+summary: Refresh source index before reindexing data stream index
+area: Data streams
+type: bug
+issues:
+ - 120314

docs/changelog/120753.yaml

@@ -0,0 +1,5 @@
+pr: 120753
+summary: Optimize `IngestDocMetadata` `isAvailable`
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/120781.yaml

@@ -0,0 +1,5 @@
+pr: 120781
+summary: Add back `keep_alive` to `async_search.submit` rest-api-spec
+area: Search
+type: bug
+issues: []

docs/changelog/120809.yaml

@@ -0,0 +1,6 @@
+pr: 120809
+summary: LTR sometines throw `NullPointerException:` Cannot read field "approximation"
+  because "top" is null
+area: Ranking
+type: bug
+issues: []

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java

@@ -90,6 +90,11 @@ public void initAgentSystemProperties(Settings settings) {
      */
     @SuppressForbidden(reason = "Need to be able to manipulate APM agent-related properties to set them dynamically")
     public void setAgentSetting(String key, String value) {
+        if (key.startsWith("global_labels.")) {
+            // Invalid agent setting, leftover from flattening global labels in APMJVMOptions
+            // https://github.com/elastic/elasticsearch/issues/120791
+            return;
+        }
         final String completeKey = "elastic.apm." + Objects.requireNonNull(key);
         AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
             if (value == null || value.isEmpty()) {
@@ -242,8 +247,8 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         return new Setting<>(qualifiedKey, "", (value) -> {
             if (qualifiedKey.equals("_na_") == false && PERMITTED_AGENT_KEYS.contains(namespace) == false) {
                 if (namespace.startsWith("global_labels.")) {
-                    // The nested labels syntax is transformed in APMJvmOptions.
-                    // Ignore these here to not fail if not correctly removed.
+                    // Invalid agent setting, leftover from flattening global labels in APMJVMOptions
+                    // https://github.com/elastic/elasticsearch/issues/120791
                     return value;
                 }
                 throw new IllegalArgumentException("Configuration [" + qualifiedKey + "] is either prohibited or unknown.");

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -1137,11 +1137,8 @@ public void testPartialRestoreSnapshotThatIncludesDataStream() {
 
     /**
      * This test is a copy of the {@link #testPartialRestoreSnapshotThatIncludesDataStream()} the only difference
-     * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
-     * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
-     * separate until this is fixed.
+     * is that one include the global state and one doesn't.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
     public void testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState() {
         final String snapshot = "test-snapshot";
         final String indexWithoutDataStream = "test-idx-no-ds";
@@ -1307,11 +1304,8 @@ public void testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices() {
 
     /**
      * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices()} ()} the only difference
-     * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
-     * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
-     * separate until this is fixed.
+     * is that one include the global state and one doesn't.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
     public void testExcludeDSFromSnapshotWhenExcludingItsIndicesWithGlobalState() {
         final String snapshot = "test-snapshot";
         final String indexWithoutDataStream = "test-idx-no-ds";
@@ -1477,10 +1471,6 @@ public void testWarningHeaderAbsentOnRestoreWithTemplates() throws Exception {
 
     }
 
-    /**
-     * This test is muted as it's awaiting the same fix as {@link #testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState()}
-     */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
     public void testWarningHeaderOnRestoreTemplateFromSnapshot() throws Exception {
         String datastreamName = "ds";