Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

.buildkite/scripts/dra-workflow.sh

@@ -22,14 +22,18 @@ if [[ "$BRANCH" == "main" ]]; then
 fi
 
 ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g")
+echo "ES_VERSION=$ES_VERSION"
 
 VERSION_SUFFIX=""
 if [[ "$WORKFLOW" == "snapshot" ]]; then
   VERSION_SUFFIX="-SNAPSHOT"
 fi
 
 BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
+echo "BEATS_BUILD_ID=$BEATS_BUILD_ID"
+
 ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
+echo "ML_CPP_BUILD_ID=$ML_CPP_BUILD_ID"
 
 LICENSE_KEY_ARG=""
 BUILD_SNAPSHOT_ARG=""

docs/changelog/114271.yaml

@@ -0,0 +1,5 @@
+pr: 114271
+summary: "[ES|QL] Skip validating remote cluster index names in parser"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/114784.yaml

@@ -0,0 +1,5 @@
+pr: 114784
+summary: "[ES|QL] make named parameter for identifier and pattern snapshot"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/114854.yaml

@@ -0,0 +1,10 @@
+pr: 114854
+summary: Adding deprecation warnings for rrf using rank and `sub_searches`
+area: Search
+type: deprecation
+issues: []
+deprecation:
+  title: Adding deprecation warnings for rrf using rank and `sub_searches`
+  area: REST API
+  details: Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter
+  impact: Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter.

docs/changelog/114869.yaml

@@ -0,0 +1,5 @@
+pr: 114869
+summary: Standardize error code when bulk body is invalid
+area: CRUD
+type: bug
+issues: []

docs/changelog/114924.yaml

@@ -0,0 +1,5 @@
+pr: 114924
+summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader`
+area: Ingest Node
+type: bug
+issues: []

docs/reference/inference/inference-apis.asciidoc

@@ -21,6 +21,7 @@ the following APIs to manage {infer} models and perform {infer}:
 * <<get-inference-api>>
 * <<post-inference-api>>
 * <<put-inference-api>>
+* <<update-inference-api>>
 
 [[inference-landscape]]
 .A representation of the Elastic inference landscape
@@ -39,6 +40,7 @@ include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::update-inference.asciidoc[]
 include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]
 include::service-anthropic.asciidoc[]

docs/reference/inference/update-inference.asciidoc

@@ -0,0 +1,87 @@
+[role="xpack"]
+[[update-inference-api]]
+=== Update inference API
+
+experimental[]
+
+Updates an {infer} endpoint.
+
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+
+
+[discrete]
+[[update-inference-api-request]]
+==== {api-request-title}
+
+`POST _inference/<inference_id>/_update`
+
+`POST _inference/<task_type>/<inference_id>/_update`
+
+
+[discrete]
+[[update-inference-api-prereqs]]
+==== {api-prereq-title}
+
+* Requires the `manage_inference` <<privileges-list-cluster,cluster privilege>> (the built-in inference_admin role grants this privilege)
+* Requires an existing {infer} endpoint, created by using the <<put-inference-api>>
+
+
+[discrete]
+[[update-inference-api-desc]]
+==== {api-description-title}
+
+The update inference API enables you to update the task_settings, secrets, and/or num_allocations of an existing {infer} endpoint.
+
+To use the update API, you can modify `task_settings`, secrets (within `service_settings`), or `num_allocations`, depending on the specific endpoint service and task_type you've created.
+To view the updatable `task_settings`, the field names of secrets (specific to each service), and the services where `num_allocations` is applicable (only for the `elasticsearch` service), refer to the following list of services available through the {infer} API.
+You will find the available task types next to each service name.
+Click the links to review the service configuration details:
+
+* <<infer-service-alibabacloud-ai-search,AlibabaCloud AI Search>> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`)
+* <<infer-service-amazon-bedrock,Amazon Bedrock>> (`completion`, `text_embedding`)
+* <<infer-service-anthropic,Anthropic>> (`completion`)
+* <<infer-service-azure-ai-studio,Azure AI Studio>> (`completion`, `text_embedding`)
+* <<infer-service-azure-openai,Azure OpenAI>> (`completion`, `text_embedding`)
+* <<infer-service-cohere,Cohere>> (`completion`, `rerank`, `text_embedding`)
+* <<infer-service-elasticsearch,Elasticsearch>> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland)
+* <<infer-service-elser,ELSER>> (`sparse_embedding`)
+* <<infer-service-google-ai-studio,Google AI Studio>> (`completion`, `text_embedding`)
+* <<infer-service-google-vertex-ai,Google Vertex AI>> (`rerank`, `text_embedding`) 
+* <<infer-service-hugging-face,Hugging Face>> (`text_embedding`)
+* <<infer-service-mistral,Mistral>> (`text_embedding`)
+* <<infer-service-openai,OpenAI>> (`completion`, `text_embedding`)
+
+
+[discrete]
+[[update-inference-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+The unique identifier of the {infer} endpoint.
+
+
+`<task_type>`::
+(Optional, string)
+The type of {infer} task that the model performs.
+Refer to the service list in the <<put-inference-api-desc,API description section>> for the available task types.
+
+
+[discrete]
+[[update-inference-api-example]]
+==== {api-examples-title}
+
+The following example shows how to update an API key of an {infer} endpoint called `my-inference-endpoint`:
+
+[source,console]
+------------------------------------------------------------
+POST _inference/my-inference-endpoint/_update
+{
+ "service_settings": {
+   "api_key": "<API_KEY>"
+ }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -139,11 +138,18 @@ void updateDatabases() throws IOException {
         if (geoipIndex != null) {
             logger.trace("The {} index is not null", GeoIpDownloader.DATABASES_INDEX);
             if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) {
-                throw new ElasticsearchException("not all primary shards of [" + DATABASES_INDEX + "] index are active");
+                logger.debug(
+                    "Not updating geoip database because not all primary shards of the [" + DATABASES_INDEX + "] index are active."
+                );
+                return;
             }
             var blockException = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, geoipIndex.getWriteIndex().getName());
             if (blockException != null) {
-                throw blockException;
+                logger.debug(
+                    "Not updating geoip database because there is a write block on the " + geoipIndex.getWriteIndex().getName() + " index",
+                    blockException
+                );
+                return;
             }
         }
         if (eagerDownloadSupplier.get() || atLeastOneGeoipProcessorSupplier.get()) {

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java

@@ -44,8 +44,7 @@
 public class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable {
 
     private static boolean includeSha256(TransportVersion version) {
-        return version.isPatchFrom(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER_BACKPORT_8_15)
-            || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER);
+        return version.isPatchFrom(TransportVersions.V_8_15_0) || version.onOrAfter(TransportVersions.ENTERPRISE_GEOIP_DOWNLOADER);
     }
 
     private static final ParseField DATABASES = new ParseField("databases");