Merge branch 'main' into enh/mapping_pre_processor2

Author: bpintea

docs/changelog/117176.yaml

@@ -0,0 +1,5 @@
+pr: 117176
+summary: Integrate IBM watsonx to Inference API for re-ranking task
+area: Experiences
+type: enhancement
+issues: []

docs/changelog/121109.yaml

@@ -0,0 +1,6 @@
+pr: 121109
+summary: Fix propagation of dynamic mapping parameter when applying `copy_to`
+area: Mapping
+type: bug
+issues:
+ - 113049

docs/changelog/121325.yaml

@@ -0,0 +1,5 @@
+pr: 121325
+summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality'
+area: Data streams
+type: bug
+issues: []

docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -286,7 +286,7 @@ protected boolean isWatcherTest() {
 
     /**
      * Compares the results of running two analyzers against many random
-     * strings. The goal is to figure out if two anlayzers are "the same" by
+     * strings. The goal is to figure out if two analyzers are "the same" by
      * comparing their results. This is far from perfect but should be fairly
      * accurate, especially for gross things like missing {@code decimal_digit}
      * token filters, and should be fairly fast because it compares a fairly

muted-tests.yml

@@ -359,12 +359,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=indices.get_alias/10_basic/Get aliases via /*/_alias/}
   issue: https://github.com/elastic/elasticsearch/issues/121290
-- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
-  method: testRerouting_HandlesTransportException_FromOtherNode
-  issue: https://github.com/elastic/elasticsearch/issues/121292
-- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
-  method: testRerouting_ToOtherNode
-  issue: https://github.com/elastic/elasticsearch/issues/121293
 - class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
   issue: https://github.com/elastic/elasticsearch/issues/121294
 - class: org.elasticsearch.env.NodeEnvironmentTests
@@ -379,6 +373,18 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/rest-api/common-options/line_125}
   issue: https://github.com/elastic/elasticsearch/issues/121338
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_751}
+  issue: https://github.com/elastic/elasticsearch/issues/121345
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testHasPrivileges
+  issue: https://github.com/elastic/elasticsearch/issues/121346
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testActivateProfile
+  issue: https://github.com/elastic/elasticsearch/issues/121151
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
+  issue: https://github.com/elastic/elasticsearch/issues/121350
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -73,4 +73,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mapping/10_copy_to.yml

@@ -0,0 +1,41 @@
+---
+copy_to from object with dynamic strict to dynamic field:
+  - requires:
+      cluster_features: ["mapper.copy_to.dynamic_handling"]
+      reason: requires a fix
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              one:
+                dynamic: strict
+                properties:
+                  k:
+                    type: keyword
+                    copy_to: two.k
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          one:
+            k: "hey"
+
+  - do:
+      search:
+        index: test
+        body:
+          docvalue_fields: [ "two.k.keyword" ]
+
+  - match:
+      hits.hits.0._source:
+        one:
+          k: "hey"
+  - match:
+      hits.hits.0.fields:
+        two.k.keyword: [ "hey" ]

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -171,6 +171,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion LINEAR_RETRIEVER_SUPPORT = def(8_837_00_0);
     public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_00_0);
     public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_00_0);
+    public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -50,6 +50,13 @@ private Wrapper(ObjectMapper parent, DocumentParserContext in) {
             this.in = in;
         }
 
+        // Used to create a copy_to context.
+        // It is important to reset `dynamic` here since it is possible that we copy into a completely different object.
+        private Wrapper(RootObjectMapper root, DocumentParserContext in) {
+            super(root, ObjectMapper.Dynamic.getRootDynamic(in.mappingLookup()), in);
+            this.in = in;
+        }
+
         @Override
         public Iterable<LuceneDocument> nonRootDocuments() {
             return in.nonRootDocuments();
@@ -711,6 +718,7 @@ in synthetic _source (to be consistent with stored _source).
 
         ContentPath path = new ContentPath();
         XContentParser parser = DotExpandingXContentParser.expandDots(new CopyToParser(copyToField, parser()), path);
+
         return new Wrapper(root(), this) {
             @Override
             public ContentPath path() {

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -30,6 +30,7 @@ public class MapperFeatures implements FeatureSpecification {
     public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed");
     public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support");
     public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix");
+    public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling");
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -45,8 +46,9 @@ public Set<NodeFeature> getTestFeatures() {
             CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX,
             META_FETCH_FIELDS_ERROR_CODE_CHANGED,
             SPARSE_VECTOR_STORE_SUPPORT,
-            SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX,
             COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT,
+            SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX,
+            DYNAMIC_HANDLING_IN_COPY_TO,
             SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE,
             ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX
         );