Merge branch 'main' into partial-result-on-demand

Author: smalyshev

docs/changelog/120128.yaml

@@ -0,0 +1,5 @@
+pr: 120128
+summary: Add Multi-Field Support for Semantic Text Fields
+area: Relevance
+type: feature
+issues: []

docs/changelog/120445.yaml

@@ -0,0 +1,5 @@
+pr: 120445
+summary: Run `GetPipelineTransportAction` on local node
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/120548.yaml

@@ -0,0 +1,5 @@
+pr: 120548
+summary: Add `remove_index_block` arg to `_create_from` api
+area: Indices APIs
+type: enhancement
+issues: []

docs/reference/mapping/types/semantic-text.asciidoc

@@ -182,16 +182,11 @@ Even if the script targets non-`semantic_text` fields, the update will fail when
 
 [discrete]
 [[copy-to-support]]
-==== `copy_to` support
+==== `copy_to` and multi-fields support
 
-The `semantic_text` field type can be the target of
-<<copy-to,`copy_to` fields>>. This means you can use a single `semantic_text`
-field to collect the values of other fields for semantic search. Each value has
-its embeddings calculated separately; each field value is a separate set of chunk(s) in
-the resulting embeddings.
-
-This imposes a restriction on bulk requests and ingestion pipelines that update documents with `semantic_text` fields.
-In these cases, all fields that are copied to a `semantic_text` field, including the `semantic_text` field value, must have a value to ensure every embedding is calculated correctly.
+The semantic_text field type can serve as the target of <<copy-to,copy_to fields>>,
+be part of a <<multi-fields,multi-field>> structure, or contain <<multi-fields,multi-fields>> internally.
+This means you can use a single field to collect the values of other fields for semantic search.
 
 For example, the following mapping:
 
@@ -201,39 +196,48 @@ PUT test-index
 {
     "mappings": {
         "properties": {
-            "infer_field": {
-                "type": "semantic_text",
-                "inference_id": ".elser-2-elasticsearch"
-            },
             "source_field": {
                 "type": "text",
                 "copy_to": "infer_field"
+            },
+            "infer_field": {
+                "type": "semantic_text",
+                "inference_id": ".elser-2-elasticsearch"
             }
         }
     }
 }
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-Will need the following bulk update request to ensure that `infer_field` is updated correctly:
+can also be declared as multi-fields:
 
 [source,console]
 ------------------------------------------------------------
-PUT test-index/_bulk
-{"update": {"_id": "1"}}
-{"doc": {"infer_field": "updated inference field", "source_field": "updated source field"}}
+PUT test-index
+{
+    "mappings": {
+        "properties": {
+            "source_field": {
+                "type": "text",
+                "fields": {
+                    "infer_field": {
+                        "type": "semantic_text",
+                        "inference_id": ".elser-2-elasticsearch"
+                    }
+                }
+            }
+        }
+    }
+}
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-Notice that both the `semantic_text` field and the source field are updated in the bulk request.
-
-
 [discrete]
 [[limitations]]
 ==== Limitations
 
 `semantic_text` field types have the following limitations:
 
 * `semantic_text` fields are not currently supported as elements of <<nested,nested fields>>.
-* `semantic_text` fields can't currently be set as part of <<dynamic-templates>>.
-* `semantic_text` fields can't be defined as <<multi-fields,multi-fields>> of another field, nor can they contain other fields as multi-fields.
+* `semantic_text` fields can't currently be set as part of <<dynamic-templates>>.

muted-tests.yml

@@ -52,21 +52,9 @@ tests:
 - class: org.elasticsearch.xpack.transform.integration.TransformIT
   method: testStopWaitForCheckpoint
   issue: https://github.com/elastic/elasticsearch/issues/106113
-- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
-  method: testPutE5Small_withPlatformAgnosticVariant
-  issue: https://github.com/elastic/elasticsearch/issues/113983
-- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
-  method: testPutE5WithTrainedModelAndInference
-  issue: https://github.com/elastic/elasticsearch/issues/114023
-- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
-  method: testPutE5Small_withPlatformSpecificVariant
-  issue: https://github.com/elastic/elasticsearch/issues/113950
 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityWithApmTracingRestIT
   method: testTracingCrossCluster
   issue: https://github.com/elastic/elasticsearch/issues/112731
-- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
-  method: testInferDeploysDefaultE5
-  issue: https://github.com/elastic/elasticsearch/issues/115361
 - class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
   method: testDeploymentSurvivesRestart {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/115528
@@ -110,9 +98,6 @@ tests:
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/10_apm/Test template reinstallation}
   issue: https://github.com/elastic/elasticsearch/issues/116445
-- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
-  method: testMultipleInferencesTriggeringDownloadAndDeploy
-  issue: https://github.com/elastic/elasticsearch/issues/117208
 - class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT
   method: testEnterpriseDownloaderTask
   issue: https://github.com/elastic/elasticsearch/issues/115163
@@ -125,9 +110,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/10_basic/Create a source only snapshot and then restore it}
   issue: https://github.com/elastic/elasticsearch/issues/117295
-- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
-  method: testInferDeploysDefaultElser
-  issue: https://github.com/elastic/elasticsearch/issues/114913
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/117027
@@ -152,9 +134,6 @@ tests:
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT
   method: testEsqlRcs2UnavailableRemoteScenarios
   issue: https://github.com/elastic/elasticsearch/issues/117419
-- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
-  method: testInferDeploysDefaultRerank
-  issue: https://github.com/elastic/elasticsearch/issues/118184
 - class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
   method: testCancelRequestWhenFailingFetchingPages
   issue: https://github.com/elastic/elasticsearch/issues/118193
@@ -251,6 +230,22 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/120482
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/120497
+- class: org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeToCharProcessorTests
+  issue: https://github.com/elastic/elasticsearch/issues/120575
+- class: org.elasticsearch.discovery.ec2.DiscoveryEc2KeystoreCredentialsIT
+  issue: https://github.com/elastic/elasticsearch/issues/120576
+- class: org.elasticsearch.discovery.ec2.DiscoveryEc2KeystoreSessionCredentialsIT
+  issue: https://github.com/elastic/elasticsearch/issues/120577
+- class: org.elasticsearch.discovery.ec2.DiscoveryEc2SystemPropertyCredentialsIT
+  issue: https://github.com/elastic/elasticsearch/issues/120578
+- class: org.elasticsearch.discovery.ec2.DiscoveryEc2EnvironmentVariableCredentialsIT
+  issue: https://github.com/elastic/elasticsearch/issues/120579
+- class: org.elasticsearch.action.search.SearchProgressActionListenerIT
+  method: testSearchProgressWithHitsAndAggs
+  issue: https://github.com/elastic/elasticsearch/issues/120583
+- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
+  method: testEnrichExplosionManyMatches
+  issue: https://github.com/elastic/elasticsearch/issues/120587
 
 # Examples:
 #

qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
 import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction;
+import org.elasticsearch.action.ingest.GetPipelineAction;
 import org.elasticsearch.action.support.CancellableActionTestPlugin;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.RefCountingListener;
@@ -103,6 +104,10 @@ public void testClusterGetSettingsCancellation() {
         runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_cluster/settings"), ClusterGetSettingsAction.NAME);
     }
 
+    public void testGetPipelineCancellation() {
+        runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_ingest/pipeline"), GetPipelineAction.NAME);
+    }
+
     private void runRestActionCancellationTest(Request request, String actionName) {
         final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY);
 

rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json

@@ -29,7 +29,7 @@
       ]
     },
     "body":{
-      "description":"The body contains the fields `mappings_override` and `settings_override`.",
+      "description":"The body contains the fields `mappings_override`, `settings_override`, and `remove_index_blocks`.",
       "required":false
     }
   }

server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/ShardLockFailureIT.java

@@ -72,7 +72,14 @@ public void testShardLockFailure() throws Exception {
             var ignored1 = internalCluster().getInstance(NodeEnvironment.class, node).shardLock(shardId, "blocked for test");
             var mockLog = MockLog.capture(IndicesClusterStateService.class);
         ) {
-
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "hot threads",
+                    "org.elasticsearch.indices.cluster.IndicesClusterStateService",
+                    Level.WARN,
+                    "[testindex][0]: acquire shard lock for create"
+                )
+            );
             mockLog.addExpectation(new MockLog.LoggingExpectation() {
                 private final CountDownLatch countDownLatch = new CountDownLatch(1);
                 int debugMessagesSeen = 0;
@@ -147,6 +154,14 @@ public void testShardLockTimeout() throws Exception {
             var ignored1 = internalCluster().getInstance(NodeEnvironment.class, node).shardLock(shardId, "blocked for test");
             var mockLog = MockLog.capture(IndicesClusterStateService.class);
         ) {
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "hot threads",
+                    "org.elasticsearch.indices.cluster.IndicesClusterStateService",
+                    Level.WARN,
+                    "[testindex][0]: acquire shard lock for create"
+                )
+            );
             mockLog.addExpectation(
                 new MockLog.SeenEventExpectation(
                     "timeout message",

server/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java

@@ -10,14 +10,18 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.support.master.MasterNodeReadRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.Map;
 
-public class GetPipelineRequest extends MasterNodeReadRequest<GetPipelineRequest> {
+public class GetPipelineRequest extends LocalClusterStateRequest {
 
     private final String[] ids;
     private final boolean summary;
@@ -35,19 +39,17 @@ public GetPipelineRequest(TimeValue masterNodeTimeout, String... ids) {
         this(masterNodeTimeout, false, ids);
     }
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     public GetPipelineRequest(StreamInput in) throws IOException {
         super(in);
         ids = in.readStringArray();
         summary = in.readBoolean();
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArray(ids);
-        out.writeBoolean(summary);
-    }
-
     public String[] getIds() {
         return ids;
     }
@@ -60,4 +62,9 @@ public boolean isSummary() {
     public ActionRequestValidationException validate() {
         return null;
     }
+
+    @Override
+    public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+        return new CancellableTask(id, type, action, "", parentTaskId, headers);
+    }
 }

server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java

@@ -11,15 +11,14 @@
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -30,16 +29,6 @@ public class GetPipelineResponse extends ActionResponse implements ToXContentObj
     private final List<PipelineConfiguration> pipelines;
     private final boolean summary;
 
-    public GetPipelineResponse(StreamInput in) throws IOException {
-        super(in);
-        int size = in.readVInt();
-        pipelines = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            pipelines.add(PipelineConfiguration.readFrom(in));
-        }
-        summary = in.readBoolean();
-    }
-
     public GetPipelineResponse(List<PipelineConfiguration> pipelines, boolean summary) {
         this.pipelines = pipelines;
         this.summary = summary;
@@ -58,6 +47,11 @@ public List<PipelineConfiguration> pipelines() {
         return Collections.unmodifiableList(pipelines);
     }
 
+    /**
+     * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+     * we no longer need to support calling this action remotely.
+     */
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(pipelines);