Merge branch 'main' into lucene_snapshot

Author: javanna

build-tools-internal/build.gradle

@@ -384,6 +384,15 @@ tasks.named("jar") {
   exclude("classpath.index")
 }
 
+spotless {
+  java {
+    // IDEs can sometimes run annotation processors that leave files in
+    // here, causing Spotless to complain. Even though this path ought not
+    // to exist, exclude it anyway in order to avoid spurious failures.
+    toggleOffOn()
+  }
+}
+
 def resolveMainWrapperVersion() {
   new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim()
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -29,11 +29,13 @@ public enum DockerBase {
     CLOUD_ESS(null, "-cloud-ess", "apt-get"),
 
     // Chainguard based wolfi image with latest jdk
-    WOLFI(
-        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0",
+    // This is usually updated via renovatebot
+    // spotless:off
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984",
         "-wolfi",
         "apk"
     ),
+    // spotless:on
 
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the Wolfi image.

build-tools-internal/src/main/resources/changelog-schema.json

@@ -284,6 +284,7 @@
         "Cluster and node setting",
         "Command line tool",
         "CRUD",
+        "ES|QL",
         "Index setting",
         "Ingest",
         "JVM option",

docs/changelog/113563.yaml

@@ -0,0 +1,5 @@
+pr: 113563
+summary: Use ELSER By Default For Semantic Text
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/113897.yaml

@@ -0,0 +1,6 @@
+pr: 113897
+summary: "Add chunking settings configuration to `CohereService,` `AmazonBedrockService,`\
+  \ and `AzureOpenAiService`"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/113967.yaml

@@ -0,0 +1,13 @@
+pr: 113967
+summary: "ESQL: Entirely remove META FUNCTIONS"
+area: ES|QL
+type: breaking
+issues: []
+breaking:
+  title: "ESQL: Entirely remove META FUNCTIONS"
+  area: ES|QL
+  details: |
+    Removes an undocumented syntax from ESQL: META FUNCTION. This was never
+    reliable or really useful. Consult the documentation instead.
+  impact: "Removes an undocumented syntax from ESQL: META FUNCTION"
+  notable: false

docs/changelog/114002.yaml

@@ -0,0 +1,5 @@
+pr: 114002
+summary: Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts
+area: Infra/Scripting
+type: enhancement
+issues: []

docs/reference/esql/multivalued-fields.asciidoc

@@ -177,6 +177,37 @@ POST /_query
 ----
 // TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
 
+[discrete]
+[[esql-multivalued-nulls]]
+==== `null` in a list
+
+`null` values in a list are not preserved at the storage layer:
+
+[source,console,id=esql-multivalued-fields-multivalued-nulls]
+----
+POST /mv/_doc?refresh
+{ "a": [2, null, 1] }
+
+POST /_query
+{
+  "query": "FROM mv | LIMIT 1"
+}
+----
+
+[source,console-result]
+----
+{
+  "took": 28,
+  "columns": [
+    { "name": "a", "type": "long"},
+  ],
+  "values": [
+    [[1, 2]],
+  ]
+}
+----
+// TESTRESPONSE[s/"took": 28/"took": "$body.took"/]
+
 [discrete]
 [[esql-multivalued-fields-functions]]
 ==== Functions

docs/reference/intro.asciidoc

@@ -204,7 +204,7 @@ For general content, you have the following options for adding data to {es} indi
 If you're building a website or app, then you can call Elasticsearch APIs using an https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} client] in the programming language of your choice. If you use the Python client, then check out the `elasticsearch-labs` repo for various https://github.com/elastic/elasticsearch-labs/tree/main/notebooks/search/python-examples[example notebooks]. 
 * {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[File upload]: Use the {kib} file uploader to index single files for one-off testing and exploration. The GUI guides you through setting up your index and field mappings.
 * https://github.com/elastic/crawler[Web crawler]: Extract and index web page content into {es} documents.
-* {enterprise-search-ref}/connectors.html[Connectors]: Sync data from various third-party data sources to create searchable, read-only replicas in {es}.
+* <<es-connectors,Connectors>>: Sync data from various third-party data sources to create searchable, read-only replicas in {es}.
 
 [discrete]
 [[es-ingestion-overview-timestamped]]
@@ -492,4 +492,4 @@ and restrictions. You can review the following guides to learn how to tune your
 * <<use-elasticsearch-for-time-series-data,Tune for time series data>>
 
 Many {es} options come with different performance considerations and trade-offs. The best way to determine the
-optimal configuration for your use case is through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[testing with your own data and queries].
+optimal configuration for your use case is through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[testing with your own data and queries].

docs/reference/reranking/index.asciidoc

@@ -1,12 +1,12 @@
 [[re-ranking-overview]]
 = Re-ranking
 
-Many search systems are built on two-stage retrieval pipelines.
+Many search systems are built on multi-stage retrieval pipelines.
 
-The first stage uses cheap, fast algorithms to find a broad set of possible matches.
+Earlier stages use cheap, fast algorithms to find a broad set of possible matches.
 
-The second stage uses a more powerful model, often machine learning-based, to reorder the documents.
-This second step is called re-ranking.
+Later stages use more powerful models, often machine learning-based, to reorder the documents.
+This step is called re-ranking.
 Because the resource-intensive model is only applied to the smaller set of pre-filtered results, this approach returns more relevant results while still optimizing for search performance and computational costs.
 
 {es} supports various ranking and re-ranking techniques to optimize search relevance and performance.
@@ -18,7 +18,7 @@ Because the resource-intensive model is only applied to the smaller set of pre-f
 
 [float]
 [[re-ranking-first-stage-pipeline]]
-=== First stage: initial retrieval
+=== Initial retrieval
 
 [float]
 [[re-ranking-ranking-overview-bm25]]
@@ -45,7 +45,7 @@ Hybrid search techniques combine results from full-text and vector search pipeli
 
 [float]
 [[re-ranking-overview-second-stage]]
-=== Second stage: Re-ranking
+=== Re-ranking
 
 When using the following advanced re-ranking pipelines, first-stage retrieval mechanisms effectively generate a set of candidates.
 These candidates are funneled into the re-ranker to perform more computationally expensive re-ranking tasks.
@@ -67,4 +67,4 @@ Learning To Rank involves training a machine learning model to build a ranking f
 LTR is best suited for when you have ample training data and need highly customized relevance tuning.
 
 include::semantic-reranking.asciidoc[]
-include::learning-to-rank.asciidoc[]
+include::learning-to-rank.asciidoc[]