Merge branch 'main' into sa-fix-and-unmute-tests

Author: slobodanadamovic

docs/changelog/117176.yaml

@@ -0,0 +1,5 @@
+pr: 117176
+summary: Integrate IBM watsonx to Inference API for re-ranking task
+area: Experiences
+type: enhancement
+issues: []

docs/changelog/121109.yaml

@@ -0,0 +1,6 @@
+pr: 121109
+summary: Fix propagation of dynamic mapping parameter when applying `copy_to`
+area: Mapping
+type: bug
+issues:
+ - 113049

docs/changelog/121325.yaml

@@ -0,0 +1,5 @@
+pr: 121325
+summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality'
+area: Data streams
+type: bug
+issues: []

muted-tests.yml

@@ -340,6 +340,27 @@ tests:
 - class: org.elasticsearch.env.NodeEnvironmentTests
   method: testGetBestDowngradeVersion
   issue: https://github.com/elastic/elasticsearch/issues/121316
+- class: org.elasticsearch.index.engine.ShuffleForcedMergePolicyTests
+  method: testDiagnostics
+  issue: https://github.com/elastic/elasticsearch/issues/121336
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/security/invalidate-tokens/line_194}
+  issue: https://github.com/elastic/elasticsearch/issues/121337
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/common-options/line_125}
+  issue: https://github.com/elastic/elasticsearch/issues/121338
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_751}
+  issue: https://github.com/elastic/elasticsearch/issues/121345
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testHasPrivileges
+  issue: https://github.com/elastic/elasticsearch/issues/121346
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testActivateProfile
+  issue: https://github.com/elastic/elasticsearch/issues/121151
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
+  issue: https://github.com/elastic/elasticsearch/issues/121350
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -73,4 +73,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mapping/10_copy_to.yml

@@ -0,0 +1,41 @@
+---
+copy_to from object with dynamic strict to dynamic field:
+  - requires:
+      cluster_features: ["mapper.copy_to.dynamic_handling"]
+      reason: requires a fix
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              one:
+                dynamic: strict
+                properties:
+                  k:
+                    type: keyword
+                    copy_to: two.k
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          one:
+            k: "hey"
+
+  - do:
+      search:
+        index: test
+        body:
+          docvalue_fields: [ "two.k.keyword" ]
+
+  - match:
+      hits.hits.0._source:
+        one:
+          k: "hey"
+  - match:
+      hits.hits.0.fields:
+        two.k.keyword: [ "hey" ]

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -171,6 +171,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion LINEAR_RETRIEVER_SUPPORT = def(8_837_00_0);
     public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_00_0);
     public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_00_0);
+    public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -50,6 +50,13 @@ private Wrapper(ObjectMapper parent, DocumentParserContext in) {
             this.in = in;
         }
 
+        // Used to create a copy_to context.
+        // It is important to reset `dynamic` here since it is possible that we copy into a completely different object.
+        private Wrapper(RootObjectMapper root, DocumentParserContext in) {
+            super(root, ObjectMapper.Dynamic.getRootDynamic(in.mappingLookup()), in);
+            this.in = in;
+        }
+
         @Override
         public Iterable<LuceneDocument> nonRootDocuments() {
             return in.nonRootDocuments();
@@ -711,6 +718,7 @@ in synthetic _source (to be consistent with stored _source).
 
         ContentPath path = new ContentPath();
         XContentParser parser = DotExpandingXContentParser.expandDots(new CopyToParser(copyToField, parser()), path);
+
         return new Wrapper(root(), this) {
             @Override
             public ContentPath path() {

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -30,6 +30,7 @@ public class MapperFeatures implements FeatureSpecification {
     public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed");
     public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support");
     public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix");
+    public static final NodeFeature DYNAMIC_HANDLING_IN_COPY_TO = new NodeFeature("mapper.copy_to.dynamic_handling");
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -45,8 +46,9 @@ public Set<NodeFeature> getTestFeatures() {
             CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX,
             META_FETCH_FIELDS_ERROR_CODE_CHANGED,
             SPARSE_VECTOR_STORE_SUPPORT,
-            SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX,
             COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT,
+            SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX,
+            DYNAMIC_HANDLING_IN_COPY_TO,
             SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE,
             ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX
         );

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.ObjectMapper;
-import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -50,11 +49,7 @@ private Supplier<Map<String, Object>> keywordMapping(
             // We only add copy_to to keywords because we get into trouble with numeric fields that are copied to dynamic fields.
             // If first copied value is numeric, dynamic field is created with numeric field type and then copy of text values fail.
             // Actual value being copied does not influence the core logic of copy_to anyway.
-            //
-            // TODO
-            // We don't use copy_to on fields that are inside an object with dynamic: strict
-            // because we'll hit https://github.com/elastic/elasticsearch/issues/113049.
-            if (request.dynamicMapping() != DynamicMapping.FORBIDDEN && ESTestCase.randomDouble() <= 0.05) {
+            if (ESTestCase.randomDouble() <= 0.05) {
                 var options = request.eligibleCopyToFields()
                     .stream()
                     .filter(f -> f.equals(request.fieldName()) == false)