Merge branch 'main' of github.com:elastic/elasticsearch into bugfix/fix-priveledges-in-system-migration-block-main

Author: JVerwolf

docs/changelog/117176.yaml

@@ -0,0 +1,5 @@
+pr: 117176
+summary: Integrate IBM watsonx to Inference API for re-ranking task
+area: Experiences
+type: enhancement
+issues: []

docs/changelog/121109.yaml

@@ -0,0 +1,6 @@
+pr: 121109
+summary: Fix propagation of dynamic mapping parameter when applying `copy_to`
+area: Mapping
+type: bug
+issues:
+ - 113049

docs/changelog/121260.yaml

@@ -0,0 +1,5 @@
+pr: 121260
+summary: Introduce a pre-mapping logical plan processing step
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/121325.yaml

@@ -0,0 +1,5 @@
+pr: 121325
+summary: '`ReindexDataStreamIndex` bug in assertion caused by reference equality'
+area: Data streams
+type: bug
+issues: []

docs/src/yamlRestTest/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -286,7 +286,7 @@ protected boolean isWatcherTest() {
 
     /**
      * Compares the results of running two analyzers against many random
-     * strings. The goal is to figure out if two anlayzers are "the same" by
+     * strings. The goal is to figure out if two analyzers are "the same" by
      * comparing their results. This is far from perfect but should be fairly
      * accurate, especially for gross things like missing {@code decimal_digit}
      * token filters, and should be fairly fast because it compares a fairly

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -31,10 +31,22 @@
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
+    public record BootstrapArgs(
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Path[] dataDirs,
+        Path configDir,
+        Path tempDir
+    ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
             requireNonNull(pluginResolver);
+            requireNonNull(dataDirs);
+            if (dataDirs.length == 0) {
+                throw new IllegalArgumentException("must provide at least one data directory");
+            }
+            requireNonNull(configDir);
+            requireNonNull(tempDir);
         }
     }
 
@@ -50,13 +62,22 @@ public static BootstrapArgs bootstrapArgs() {
      *
      * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
+     * @param dataDirs data directories for Elasticsearch
+     * @param configDir the config directory for Elasticsearch
+     * @param tempDir the temp directory for Elasticsearch
      */
-    public static void bootstrap(Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(
+        Map<String, Policy> pluginPolicies,
+        Function<Class<?>, String> pluginResolver,
+        Path[] dataDirs,
+        Path configDir,
+        Path tempDir
+    ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver);
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir);
         exportInitializationToAgent();
         loadAgent(findAgentJar());
         selfTest();

muted-tests.yml

@@ -231,9 +231,6 @@ tests:
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/140_data_stream_aliases/Create data stream aliases using wildcard expression}
   issue: https://github.com/elastic/elasticsearch/issues/120890
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/*}
-  issue: https://github.com/elastic/elasticsearch/issues/120816
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
   method: testAuthenticateShouldNotFallThroughInCaseOfFailure
   issue: https://github.com/elastic/elasticsearch/issues/120902
@@ -330,6 +327,10 @@ tests:
   method: testCrossClusterAsyncQueryStop
   issue: https://github.com/elastic/elasticsearch/issues/121249
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/*}
+  issue: https://github.com/elastic/elasticsearch/issues/120816
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/*}
   issue: https://github.com/elastic/elasticsearch/issues/120816
 - class: org.elasticsearch.upgrades.VectorSearchIT
   method: testBBQVectorSearch {upgradedNodes=0}
@@ -359,17 +360,38 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=indices.get_alias/10_basic/Get aliases via /*/_alias/}
   issue: https://github.com/elastic/elasticsearch/issues/121290
-- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
-  method: testRerouting_HandlesTransportException_FromOtherNode
-  issue: https://github.com/elastic/elasticsearch/issues/121292
-- class: org.elasticsearch.xpack.inference.action.TransportInferenceActionTests
-  method: testRerouting_ToOtherNode
-  issue: https://github.com/elastic/elasticsearch/issues/121293
-- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
-  issue: https://github.com/elastic/elasticsearch/issues/121294
 - class: org.elasticsearch.env.NodeEnvironmentTests
   method: testGetBestDowngradeVersion
   issue: https://github.com/elastic/elasticsearch/issues/121316
+- class: org.elasticsearch.index.engine.ShuffleForcedMergePolicyTests
+  method: testDiagnostics
+  issue: https://github.com/elastic/elasticsearch/issues/121336
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/security/invalidate-tokens/line_194}
+  issue: https://github.com/elastic/elasticsearch/issues/121337
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/rest-api/common-options/line_125}
+  issue: https://github.com/elastic/elasticsearch/issues/121338
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_751}
+  issue: https://github.com/elastic/elasticsearch/issues/121345
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testHasPrivileges
+  issue: https://github.com/elastic/elasticsearch/issues/121346
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testActivateProfile
+  issue: https://github.com/elastic/elasticsearch/issues/121151
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
+  issue: https://github.com/elastic/elasticsearch/issues/121350
+- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
+  method: test {p0=search.vectors/42_knn_search_int4_flat/KNN Vector similarity search only}
+  issue: https://github.com/elastic/elasticsearch/issues/121395
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  issue: https://github.com/elastic/elasticsearch/issues/121407
+- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
+  method: testDependentVariableIsAliasToNested
+  issue: https://github.com/elastic/elasticsearch/issues/121415
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -73,4 +73,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("index/92_metrics_auto_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
+  task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mapping/10_copy_to.yml

@@ -0,0 +1,41 @@
+---
+copy_to from object with dynamic strict to dynamic field:
+  - requires:
+      cluster_features: ["mapper.copy_to.dynamic_handling"]
+      reason: requires a fix
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              one:
+                dynamic: strict
+                properties:
+                  k:
+                    type: keyword
+                    copy_to: two.k
+
+  - do:
+      index:
+        index: test
+        id: 1
+        refresh: true
+        body:
+          one:
+            k: "hey"
+
+  - do:
+      search:
+        index: test
+        body:
+          docvalue_fields: [ "two.k.keyword" ]
+
+  - match:
+      hits.hits.0._source:
+        one:
+          k: "hey"
+  - match:
+      hits.hits.0.fields:
+        two.k.keyword: [ "hey" ]

server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingIT.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.indices.mapping.put;
+
+import org.apache.logging.log4j.Level;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.MockLog;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PutMappingIT extends ESSingleNodeTestCase {
+
+    @TestLogging(
+        reason = "testing DEBUG logging",
+        value = "org.elasticsearch.action.admin.indices.mapping.put.TransportPutMappingAction:DEBUG"
+    )
+    public void testFailureLogging() {
+        final var indexName = randomIdentifier();
+        createIndex(indexName);
+        final var fieldName = randomIdentifier();
+        safeGet(client().execute(TransportPutMappingAction.TYPE, new PutMappingRequest(indexName).source(fieldName, "type=keyword")));
+        MockLog.assertThatLogger(
+            () -> assertThat(
+                asInstanceOf(
+                    IllegalArgumentException.class,
+                    safeAwaitFailure(
+                        AcknowledgedResponse.class,
+                        l -> client().execute(
+                            TransportPutMappingAction.TYPE,
+                            new PutMappingRequest(indexName).source(fieldName, "type=long"),
+                            l
+                        )
+                    )
+                ).getMessage(),
+                equalTo("mapper [" + fieldName + "] cannot be changed from type [keyword] to [long]")
+            ),
+            TransportPutMappingAction.class,
+            new MockLog.SeenEventExpectation(
+                "failure message",
+                TransportPutMappingAction.class.getCanonicalName(),
+                Level.DEBUG,
+                "failed to put mappings on indices [[" + indexName
+            )
+        );
+    }
+}