Merge branch 'main' into feature/logsdb-ignore-dynamic-beyond-limit

Author: salvatore-campagna

docs/changelog/113975.yaml

@@ -0,0 +1,19 @@
+pr: 113975
+summary: JDK locale database change
+area: Mapping
+type: breaking
+issues: []
+breaking:
+  title: JDK locale database change
+  area: Mapping
+  details: |
+    {es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes the locale database that is used by Elasticsearch from the COMPAT database to the CLDR database. This change can cause significant differences to the textual date formats accepted by Elasticsearch, and to calculated week-dates.
+
+    If you run {es} 8.16 on JDK version 22 or below, it will use the COMPAT locale database to match the behavior of 8.15. However, starting with {es} 9.0, {es} will use the CLDR database regardless of JDK version it is run on.
+  impact: |
+    This affects you if you use custom date formats using textual or week-date field specifiers. If you use date fields or calculated week-dates that change between the COMPAT and CLDR databases, then this change will cause Elasticsearch to reject previously valid date fields as invalid data. You might need to modify your ingest or output integration code to account for the differences between these two JDK versions.
+
+    Starting in version 8.15.2, Elasticsearch will log deprecation warnings if you are using date format specifiers that might change on upgrading to JDK 23. These warnings are visible in Kibana.
+
+    For detailed guidance, refer to <<custom-date-format-locales,Differences in locale information between JDK versions>> and the https://ela.st/jdk-23-locales[Elastic blog].
+  notable: true

docs/changelog/114665.yaml

@@ -0,0 +1,6 @@
+pr: 114665
+summary: Fixing remote ENRICH by pushing the Enrich inside `FragmentExec`
+area: ES|QL
+type: bug
+issues:
+  - 105095

docs/changelog/115314.yaml

@@ -0,0 +1,5 @@
+pr: 115314
+summary: Only aggregations require at least one shard request
+area: Search
+type: enhancement
+issues: []

docs/changelog/115594.yaml

@@ -0,0 +1,6 @@
+pr: 115594
+summary: Update `BlobCacheBufferedIndexInput::readVLong` to correctly handle negative
+  long values
+area: Search
+type: bug
+issues: []

docs/reference/inference/inference-apis.asciidoc

@@ -19,6 +19,7 @@ the following APIs to manage {infer} models and perform {infer}:
 * <<get-inference-api>>
 * <<post-inference-api>>
 * <<put-inference-api>>
+* <<stream-inference-api>>
 * <<update-inference-api>>
 
 [[inference-landscape]]
@@ -56,6 +57,7 @@ include::delete-inference.asciidoc[]
 include::get-inference.asciidoc[]
 include::post-inference.asciidoc[]
 include::put-inference.asciidoc[]
+include::stream-inference.asciidoc[]
 include::update-inference.asciidoc[]
 include::service-alibabacloud-ai-search.asciidoc[]
 include::service-amazon-bedrock.asciidoc[]

docs/reference/inference/stream-inference.asciidoc

@@ -0,0 +1,122 @@
+[role="xpack"]
+[[stream-inference-api]]
+=== Stream inference API
+
+Streams a chat completion response.
+
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face.
+For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models.
+However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
+
+
+[discrete]
+[[stream-inference-api-request]]
+==== {api-request-title}
+
+`POST /_inference/<inference_id>/_stream`
+
+`POST /_inference/<task_type>/<inference_id>/_stream`
+
+
+[discrete]
+[[stream-inference-api-prereqs]]
+==== {api-prereq-title}
+
+* Requires the `monitor_inference` <<privileges-list-cluster,cluster privilege>>
+(the built-in `inference_admin` and `inference_user` roles grant this privilege)
+* You must use a client that supports streaming.
+
+
+[discrete]
+[[stream-inference-api-desc]]
+==== {api-description-title}
+
+The stream {infer} API enables real-time responses for completion tasks by delivering answers incrementally, reducing response times during computation.
+It only works with the `completion` task type.
+
+
+[discrete]
+[[stream-inference-api-path-params]]
+==== {api-path-parms-title}
+
+`<inference_id>`::
+(Required, string)
+The unique identifier of the {infer} endpoint.
+
+
+`<task_type>`::
+(Optional, string)
+The type of {infer} task that the model performs.
+
+
+[discrete]
+[[stream-inference-api-request-body]]
+==== {api-request-body-title}
+
+`input`::
+(Required, string or array of strings)
+The text on which you want to perform the {infer} task.
+`input` can be a single string or an array.
++
+--
+[NOTE]
+====
+Inference endpoints for the `completion` task type currently only support a
+single string as input.
+====
+--
+
+
+[discrete]
+[[stream-inference-api-example]]
+==== {api-examples-title}
+
+The following example performs a completion on the example question with streaming.
+
+
+[source,console]
+------------------------------------------------------------
+POST _inference/completion/openai-completion/_stream
+{
+  "input": "What is Elastic?"
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
+The API returns the following response:
+
+
+[source,txt]
+------------------------------------------------------------
+event: message
+data: {
+  "completion":[{
+    "delta":"Elastic"
+  }]
+}
+
+event: message
+data: {
+  "completion":[{
+    "delta":" is"
+    },
+    {
+    "delta":" a"
+    }
+  ]
+}
+
+event: message
+data: {
+  "completion":[{
+    "delta":" software"
+  },
+  {
+    "delta":" company"
+  }]
+}
+
+(...)
+------------------------------------------------------------
+// NOTCONSOLE

docs/reference/mapping/params/format.asciidoc

@@ -34,13 +34,13 @@ down to the nearest day.
 Completely customizable date formats are supported. The syntax for these is explained in
 https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.html[DateTimeFormatter docs].
 
-Note that whilst the built-in formats for week dates use the ISO definition of weekyears,
+Note that while the built-in formats for week dates use the ISO definition of weekyears,
 custom formatters using the `Y`, `W`, or `w` field specifiers use the JDK locale definition
 of weekyears. This can result in different values between the built-in formats and custom formats
 for week dates.
 
 [[built-in-date-formats]]
-==== Built In Formats
+==== Built-in formats
 
 Most of the below formats have a `strict` companion format, which means that
 year, month and day parts of the month must use respectively 4, 2 and 2 digits

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -412,7 +412,7 @@ public void testSkippingShards() throws Exception {
             assertResponse(client().search(searchRequest), searchResponse -> {
                 ElasticsearchAssertions.assertNoSearchHits(searchResponse);
                 assertThat(searchResponse.getTotalShards(), equalTo(2));
-                assertThat(searchResponse.getSkippedShards(), equalTo(1));
+                assertThat(searchResponse.getSkippedShards(), equalTo(2));
                 assertThat(searchResponse.getSuccessfulShards(), equalTo(2));
             });
         }

muted-tests.yml

@@ -23,9 +23,6 @@ tests:
 - class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests
   method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications
   issue: https://github.com/elastic/elasticsearch/issues/110789
-- class: org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFileTests
-  method: testCacheFileCreatedAsSparseFile
-  issue: https://github.com/elastic/elasticsearch/issues/110801
 - class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests
   method: testSystemPropertyDisabled
   issue: https://github.com/elastic/elasticsearch/issues/110949
@@ -276,6 +273,12 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/esql/esql-across-clusters/line_197}
   issue: https://github.com/elastic/elasticsearch/issues/115575
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search}
+  issue: https://github.com/elastic/elasticsearch/issues/115600
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=indices.create/10_basic/Create lookup index}
+  issue: https://github.com/elastic/elasticsearch/issues/115605
 
 # Examples:
 #

qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
@@ -580,13 +581,14 @@ public void testSortByField() throws Exception {
 
     public void testSortByFieldOneClusterHasNoResults() throws Exception {
         assumeMultiClusterSetup();
-        // set to a value greater than the number of shards to avoid differences due to the skipping of shards
+        // setting aggs to avoid differences due to the skipping of shards when matching none
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
         boolean onlyRemote = randomBoolean();
         sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME));
         sourceBuilder.sort("type.keyword", SortOrder.ASC);
         sourceBuilder.sort("creationDate", SortOrder.DESC);
         sourceBuilder.sort("user.keyword", SortOrder.ASC);
+        sourceBuilder.aggregation(AggregationBuilders.max("max").field("creationDate"));
         CheckedConsumer<ObjectPath, IOException> responseChecker = response -> {
             assertHits(response);
             int size = response.evaluateArraySize("hits.hits");