Merge branch 'main' into rename-unpromotables

Author: elasticmachine

.buildkite/scripts/dra-workflow.sh

@@ -6,7 +6,7 @@ WORKFLOW="${DRA_WORKFLOW:-snapshot}"
 BRANCH="${BUILDKITE_BRANCH:-}"
 
 # Don't publish main branch to staging
-if [[ "$BRANCH" == *.x && "$WORKFLOW" == "staging" ]]; then
+if [[ ("$BRANCH" == "main" || "$BRANCH" == *.x) && "$WORKFLOW" == "staging" ]]; then
   exit 0
 fi
 

docs/changelog/117176.yaml

@@ -0,0 +1,5 @@
+pr: 117176
+summary: Integrate IBM watsonx to Inference API for re-ranking task
+area: Experiences
+type: enhancement
+issues: []

docs/changelog/121109.yaml

@@ -0,0 +1,6 @@
+pr: 121109
+summary: Fix propagation of dynamic mapping parameter when applying `copy_to`
+area: Mapping
+type: bug
+issues:
+ - 113049

docs/changelog/121260.yaml

@@ -0,0 +1,5 @@
+pr: 121260
+summary: Introduce a pre-mapping logical plan processing step
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/121325.yaml

@@ -0,0 +1,5 @@
+pr: 121325
+summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality'
+area: Data streams
+type: bug
+issues: []

docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -286,7 +286,7 @@ protected boolean isWatcherTest() {
 
     /**
      * Compares the results of running two analyzers against many random
-     * strings. The goal is to figure out if two anlayzers are "the same" by
+     * strings. The goal is to figure out if two analyzers are "the same" by
      * comparing their results. This is far from perfect but should be fairly
      * accurate, especially for gross things like missing {@code decimal_digit}
      * token filters, and should be fairly fast because it compares a fairly

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -31,10 +31,22 @@
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
+    public record BootstrapArgs(
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Path[] dataDirs,
+        Path configDir,
+        Path tempDir
+    ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(pluginResolver);
+            requireNonNull(dataDirs);
+            if (dataDirs.length == 0) {
+                throw new IllegalArgumentException("must provide at least one data directory");
+            }
+            requireNonNull(configDir);
+            requireNonNull(tempDir);
         }
     }
 
@@ -50,13 +62,22 @@ public static BootstrapArgs bootstrapArgs() {
      *
      * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
+     * @param dataDirs data directories for Elasticsearch
+     * @param configDir the config directory for Elasticsearch
+     * @param tempDir the temp directory for Elasticsearch
      */
-    public static void bootstrap(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Path[] dataDirs,
+        Path configDir,
+        Path tempDir
+    ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver);
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir);
         exportInitializationToAgent();
         loadAgent(findAgentJar());
         selfTest();

muted-tests.yml

@@ -231,9 +231,6 @@ tests:
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/140_data_stream_aliases/Create data stream aliases using wildcard expression}
   issue: https://github.com/elastic/elasticsearch/issues/120890
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/*}
-  issue: https://github.com/elastic/elasticsearch/issues/120816
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
   method: testAuthenticateShouldNotFallThroughInCaseOfFailure
   issue: https://github.com/elastic/elasticsearch/issues/120902
@@ -267,9 +264,6 @@ tests:
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/140_data_stream_aliases/Create data stream alias with filter}
   issue: https://github.com/elastic/elasticsearch/issues/121014
-- class: org.elasticsearch.xpack.esql.parser.StatementParserTests
-  method: testNamedFunctionArgumentInMap
-  issue: https://github.com/elastic/elasticsearch/issues/121020
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testSuggestProfilesWithName
   issue: https://github.com/elastic/elasticsearch/issues/121022
@@ -333,6 +327,10 @@ tests:
   method: testCrossClusterAsyncQueryStop
   issue: https://github.com/elastic/elasticsearch/issues/121249
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/*}
+  issue: https://github.com/elastic/elasticsearch/issues/120816
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/*}
   issue: https://github.com/elastic/elasticsearch/issues/120816
 - class: org.elasticsearch.upgrades.VectorSearchIT
   method: testBBQVectorSearch {upgradedNodes=0}
@@ -362,17 +360,35 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=indices.get_alias/10_basic/Get aliases via /*/_alias/}
   issue: https://github.com/elastic/elasticsearch/issues/121290
-- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
-  method: testRerouting_HandlesTransportException_FromOtherNode
-  issue: https://github.com/elastic/elasticsearch/issues/121292
-- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
-  method: testRerouting_ToOtherNode
-  issue: https://github.com/elastic/elasticsearch/issues/121293
-- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
-  issue: https://github.com/elastic/elasticsearch/issues/121294
 - class: org.elasticsearch.env.NodeEnvironmentTests
   method: testGetBestDowngradeVersion
   issue: https://github.com/elastic/elasticsearch/issues/121316
+- class: org.elasticsearch.index.engine.ShuffleForcedMergePolicyTests
+  method: testDiagnostics
+  issue: https://github.com/elastic/elasticsearch/issues/121336
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/security/invalidate-tokens/line_194}
+  issue: https://github.com/elastic/elasticsearch/issues/121337
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/common-options/line_125}
+  issue: https://github.com/elastic/elasticsearch/issues/121338
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_751}
+  issue: https://github.com/elastic/elasticsearch/issues/121345
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testHasPrivileges
+  issue: https://github.com/elastic/elasticsearch/issues/121346
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testActivateProfile
+  issue: https://github.com/elastic/elasticsearch/issues/121151
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
+  issue: https://github.com/elastic/elasticsearch/issues/121350
+- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
+  method: test {p0=search.vectors/42_knn_search_int4_flat/KNN Vector similarity search only}
+  issue: https://github.com/elastic/elasticsearch/issues/121395
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/121407
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -73,4 +73,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mapping/10_copy_to.yml

@@ -0,0 +1,41 @@
+---
+copy_to from object with dynamic strict to dynamic field:
+  - requires:
+      cluster_features: ["mapper.copy_to.dynamic_handling"]
+      reason: requires a fix
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              one:
+                dynamic: strict
+                properties:
+                  k:
+                    type: keyword
+                    copy_to: two.k
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          one:
+            k: "hey"
+
+  - do:
+      search:
+        index: test
+        body:
+          docvalue_fields: [ "two.k.keyword" ]
+
+  - match:
+      hits.hits.0._source:
+        one:
+          k: "hey"
+  - match:
+      hits.hits.0.fields:
+        two.k.keyword: [ "hey" ]