Merge branch 'main' into lucene_snapshot

Author: ChrisHegarty

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.9", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -302,8 +302,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.9 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.9
+      - label: "{{matrix.image}} / 8.17.10 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.10
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -316,7 +316,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.9
+          BWC_VERSION: 8.17.10
 
       - label: "{{matrix.image}} / 8.18.5 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.5

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.9 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.9#bwcTest
+      - label: 8.17.10 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.9
+          BWC_VERSION: 8.17.10
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.9", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.9", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -16,7 +16,7 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.9"
+  - "8.17.10"
   - "8.18.5"
   - "8.19.1"
   - "9.0.5"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.17.9"
+  - "8.17.10"
   - "8.18.5"
   - "8.19.1"
   - "9.0.5"

docs/changelog/132162.yaml

@@ -0,0 +1,6 @@
+pr: 132162
+summary: Fix default missing index sort value of `data_nanos` pre 7.14
+area: Search
+type: bug
+issues:
+ - 132040

docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md

@@ -91,3 +91,37 @@ The `linear` retriever supports the following normalizers:
     score = (score - min) / (max - min)
     ```
 * `l2_norm`: Normalizes scores using the L2 norm of the score values {applies_to}`stack: ga 9.1`
+
+## Example
+
+This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking.
+
+```console
+GET my_index/_search
+{
+  "retriever": {
+    "linear": {
+      "retrievers": [
+        {
+          "retriever": {
+            "knn": {
+              ...
+            }
+          },
+          "weight": 5 # KNN query weighted 5x
+        },
+        {
+          "retriever": {
+            "standard": {
+              ...
+            }
+          },
+          "weight": 1.5 # BM25 query weighted 1.5x
+        }
+      ]
+    }
+  }
+}
+```
+
+See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever).

docs/release-notes/index.md

@@ -361,6 +361,7 @@ Machine Learning:
 * Mark token pruning for sparse vector as GA [#128854](https://github.com/elastic/elasticsearch/pull/128854)
 * Move to the Cohere V2 API for new inference endpoints [#129884](https://github.com/elastic/elasticsearch/pull/129884)
 * Semantic Text Chunking Indexing Pressure [#125517](https://github.com/elastic/elasticsearch/pull/125517)
+* Track memory used in the hierarchical results normalizer [#2831](https://github.com/elastic/ml-cpp/pull/2831)
 * Upgrade AWS v2 SDK to 2.30.38 [#124738](https://github.com/elastic/elasticsearch/pull/124738)
 * [Inference API] Propagate product use case http header to EIS [#124025](https://github.com/elastic/elasticsearch/pull/124025)
 * [ML] Add HuggingFace Chat Completion support to the Inference Plugin [#127254](https://github.com/elastic/elasticsearch/pull/127254)

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ES92Int7VectorsScorer.java

@@ -41,6 +41,13 @@ public ES92Int7VectorsScorer(IndexInput in, int dimensions) {
         this.dimensions = dimensions;
     }
 
+    /**
+     * Checks if the current implementation supports fast native access.
+     */
+    public boolean hasNativeAccess() {
+        return false; // This class does not support native access
+    }
+
     /**
      * compute the quantize distance between the provided quantized query and the quantized vector
      * that is read from the wrapped {@link IndexInput}.