Merge branch 'main' of github.com:elastic/elasticsearch into diskbbq_segment_affinity

Author: tteofili

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -334,8 +334,8 @@ steps:
         env:
           BWC_VERSION: 8.18.5
 
-      - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
+      - label: "{{matrix.image}} / 8.19.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,7 +348,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.0
+          BWC_VERSION: 8.19.1
 
       - label: "{{matrix.image}} / 9.0.5 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.5

.buildkite/pipelines/periodic.yml

@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.0#bwcTest
+      - label: 8.19.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.0
+          BWC_VERSION: 8.19.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -18,7 +18,7 @@ BWC_VERSION:
   - "8.16.6"
   - "8.17.10"
   - "8.18.5"
-  - "8.19.0"
+  - "8.19.1"
   - "9.0.5"
   - "9.1.1"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,7 @@
 BWC_VERSION:
   - "8.17.10"
   - "8.18.5"
-  - "8.19.0"
+  - "8.19.1"
   - "9.0.5"
   - "9.1.1"
   - "9.2.0"

docs/changelog/132088.yaml

@@ -0,0 +1,5 @@
+pr: 132088
+summary: Fix combine result for `ingest_took`
+area: ES|QL
+type: bug
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md

@@ -91,3 +91,37 @@ The `linear` retriever supports the following normalizers:
     score = (score - min) / (max - min)
     ```
 * `l2_norm`: Normalizes scores using the L2 norm of the score values {applies_to}`stack: ga 9.1`
+
+## Example
+
+This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking.
+
+```console
+GET my_index/_search
+{
+  "retriever": {
+    "linear": {
+      "retrievers": [
+        {
+          "retriever": {
+            "knn": {
+              ...
+            }
+          },
+          "weight": 5 # KNN query weighted 5x
+        },
+        {
+          "retriever": {
+            "standard": {
+              ...
+            }
+          },
+          "weight": 1.5 # BM25 query weighted 1.5x
+        }
+      ]
+    }
+  }
+}
+```
+
+See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever).

muted-tests.yml

@@ -290,7 +290,7 @@ tests:
   method: testOneRemoteClusterPartial
   issue: https://github.com/elastic/elasticsearch/issues/124055
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyOnTheLookupIndex}
   issue: https://github.com/elastic/elasticsearch/issues/128030
 - class: org.elasticsearch.packaging.test.EnrollmentProcessTests
   method: test20DockerAutoFormCluster
@@ -314,11 +314,8 @@ tests:
   method: testEarlyTermination
   issue: https://github.com/elastic/elasticsearch/issues/128721
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {lookup-join.EnrichLookupStatsBug ASYNC}
+  method: test {csv-spec:lookup-join.EnrichLookupStatsBug}
   issue: https://github.com/elastic/elasticsearch/issues/129228
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
-  method: test {lookup-join.EnrichLookupStatsBug SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/129229
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {lookup-join.MultipleBatches*
   issue: https://github.com/elastic/elasticsearch/issues/129210
@@ -363,7 +360,7 @@ tests:
   method: testWriteCanRejectOnPrimaryBasedOnMaxOperationSize
   issue: https://github.com/elastic/elasticsearch/issues/130281
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyOnFrom SYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyOnFrom}
   issue: https://github.com/elastic/elasticsearch/issues/130296
 - class: org.elasticsearch.xpack.esql.inference.bulk.BulkInferenceExecutorTests
   method: testSuccessfulExecution
@@ -378,7 +375,7 @@ tests:
   method: failed tasks output logged to console when spooling true
   issue: https://github.com/elastic/elasticsearch/issues/119509
 - class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyFromRow ASYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyFromRow}
   issue: https://github.com/elastic/elasticsearch/issues/130642
 - class: org.elasticsearch.indices.stats.IndexStatsIT
   method: testFilterCacheStats
@@ -390,7 +387,7 @@ tests:
   method: testOpenContextsAfterRejections
   issue: https://github.com/elastic/elasticsearch/issues/130821
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: test {lookup-join.MvJoinKeyOnFromAfterStats ASYNC}
+  method: test {csv-spec:lookup-join.MvJoinKeyOnFromAfterStats}
   issue: https://github.com/elastic/elasticsearch/issues/131148
 - class: org.elasticsearch.xpack.esql.ccq.MultiClustersIT
   method: testLookupJoinAliases
@@ -533,6 +530,18 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/132273
+- class: org.elasticsearch.common.logging.JULBridgeTests
+  method: testThrowable
+  issue: https://github.com/elastic/elasticsearch/issues/132280
+- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
+  method: test {p0=data_stream/10_basic/Create hidden data stream with match all template}
+  issue: https://github.com/elastic/elasticsearch/issues/132298
+- class: org.elasticsearch.xpack.ml.integration.AutodetectMemoryLimitIT
+  method: testManyDistinctOverFields
+  issue: https://github.com/elastic/elasticsearch/issues/132308
+- class: org.elasticsearch.xpack.ml.integration.AutodetectMemoryLimitIT
+  method: testTooManyByAndOverFields
+  issue: https://github.com/elastic/elasticsearch/issues/132310
 
 # Examples:
 #
@@ -564,7 +573,6 @@ tests:
 #    issue: "https://github.com/elastic/elasticsearch/..."
 #  Note that this mutes for the unit-test-like CsvTests only.
 #  Muting all the integration tests can be done using the class "org.elasticsearch.xpack.esql.**".
-#  Consider however, that some tests are named as "test {file.test SYNC}" and "ASYNC" in the integration tests.
 #  To mute all 3 tests safely everywhere use:
 #  - class: "org.elasticsearch.xpack.esql.**"
 #    method: "test {union_types.MultiIndexIpStringStatsInline}"

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -213,6 +213,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN_8_19 = def(8_841_0_62);
     public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_8_19 = def(8_841_0_63);
     public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64);
+    public static final TransportVersion INITIAL_ELASTICSEARCH_8_19_1 = def(8_841_0_65);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);

server/src/main/java/org/elasticsearch/Version.java

@@ -213,6 +213,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_18_4 = new Version(8_18_04_99);
     public static final Version V_8_18_5 = new Version(8_18_05_99);
     public static final Version V_8_19_0 = new Version(8_19_00_99);
+    public static final Version V_8_19_1 = new Version(8_19_01_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);
     public static final Version V_9_0_1 = new Version(9_00_01_99);
     public static final Version V_9_0_2 = new Version(9_00_02_99);