Merge branch 'main' into fix/134809-1

Author: prwhelan

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -90,6 +90,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertUpperBound("9.2", "new_tv,8124000")
     }
 
+    /*
+    temporarily muted, see https://github.com/elastic/elasticsearch/pull/135226
+
     def "invalid changes to a upper bounds should be reverted"() {
         given:
         transportVersionUpperBound("9.2", "modification", "9000000")
@@ -144,7 +147,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertReferableDefinitionDoesNotExist("test_tv")
         assertUpperBound("9.2", "existing_92,8123000")
         assertUpperBound("9.1", "existing_92,8012001")
-    }
+    }*/
 
     def "a reference can be renamed"() {
         given:
@@ -242,8 +245,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
     def "unreferenced definitions are removed"() {
         given:
         referableTransportVersion("test_tv", "8124000,8012002")
+        /*
+        TODO: reset of upper bounds
         transportVersionUpperBound("9.2", "test_tv", "8124000")
         transportVersionUpperBound("9.1", "test_tv", "8012002")
+         */
 
         when:
         def result = runGenerateAndValidateTask().build()
@@ -406,6 +412,8 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertUpperBound("9.2", "new_tv,8124000")
     }
 
+    /*
+    TODO: reset of upper bounds
     def "deleted upper bounds files are restored"() {
         given:
         file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete()
@@ -416,7 +424,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         then:
         assertGenerateAndValidateSuccess(result)
         assertUpperBound("9.2", "existing_92,8123000")
-    }
+    }*/
 
     def "upper bounds files must exist for backport branches"() {
         when:

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -103,7 +103,8 @@ public void run() throws IOException {
 
         getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
         if (targetDefinitionName.isEmpty()) {
-            resetAllUpperBounds(resources);
+            // TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
+            // resetAllUpperBounds(resources);
         } else {
             List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
             // (Re)write the definition file.

docs/changelog/135247.yaml

@@ -0,0 +1,5 @@
+pr: 135247
+summary: Fix alias id when drop all aggregates
+area: ES|QL
+type: bug
+issues: []

docs/changelog/135263.yaml

@@ -0,0 +1,5 @@
+pr: 135263
+summary: Optimize `dotCount` in expanding dot parser
+area: "Mapping"
+type: enhancement
+issues: []

docs/changelog/135270.yaml

@@ -0,0 +1,5 @@
+pr: 135270
+summary: Add .reindexed-v7-ml-anomalies-* to anomaly results template index pattern
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/135299.yaml

@@ -0,0 +1,47 @@
+pr: 135299
+summary: Release DiskBBQ(`bbq_disk`) index type for `dense_vector` fields
+area: Vector Search
+type: feature
+issues: []
+highlight:
+  title: Release DiskBBQ(`bbq_disk`) index type for `dense_vector` fields
+  body: |-
+     This provides a new index type called DiskBBQ (`bbq_disk`).
+     DiskBBQ is a cluster based format that provides:
+       - faster and cheaper indexing than HNSW
+       - Better behavior in lower memory environments (degrades linearly, not exponentially)
+       - Is near HNSW for QPS when the index is in memory
+
+     Current restrictions:
+       - only floating point values are allowed currently
+       - quantization is only to a single bit, so not recommended for low dimensionality vectors
+       - all other restrictions that exist for `dense_vector` fields still apply
+
+     To utilize the format, its just like any other:
+     [source,yaml]
+     ----------------------------
+     PUT vectors
+      {
+        "mappings": {
+          "properties": {
+            "vector": {"type": "dense_vector", "index_options": {"type": "disk_bbq"}
+          }
+        }
+      }
+     ----------------------------
+      Querying is just like any other field.
+     [source,yaml]
+     ----------------------------
+     POST vectors/_search{
+       "query": {
+         "knn": {
+           "field": "vector",
+           "query_vector": <vector>,
+           "k": 3
+         }
+       }
+     }
+     ----------------------------
+     `num_candidates` can be used for tuning approximate nature of the search.
+     Or, more granular control can be provided by setting `visit_percentage` directly.
+  notable: true

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -341,18 +341,19 @@ $$$dense-vector-index-options$$$
 `type`
 :   (Required, string) The type of kNN algorithm to use. Can be either any of:
     * `hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) for scalable approximate kNN search. This supports all `element_type` values.
-    * `int8_hnsw` - The default index type for some float vectors:      
+    * `int8_hnsw` - The default index type for some float vectors:
       * {applies_to}`stack: ga 9.1` Default for float vectors with less than 384 dimensions.
       * {applies_to}`stack: ga 9.0` Default for float all vectors.
       This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 4x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization).
     * `int4_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 8x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization).
     * `bbq_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically binary quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 32x at the cost of accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization).
-      
+
       {applies_to}`stack: ga 9.1` `bbq_hnsw` is the default index type for float vectors with greater than or equal to 384 dimensions.
     * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values.
-    * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`.
-    * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports `element_type` of `float`.
-    * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatically binary quantization. Only supports `element_type` of `float`.
+    * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatic scalar quantization. Only supports `element_type` of `float`.
+    * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatic half-byte scalar quantization. Only supports `element_type` of `float`.
+    * `bbq_flat` - This utilizes a brute-force search algorithm in addition to automatic binary quantization. Only supports `element_type` of `float`.
+    * {applies_to}`stack: ga 9.2` `bbq_disk` - This utilizes a variant of [k-means clustering algorithm](https://en.wikipedia.org/wiki/K-means_clustering) in addition to automatic binary quantization to partition vectors and search subspaces rather than an entire graph structure as in with HNSW. Only supports `element_type` of `float`.  This combines the benefits of BBQ quantization with partitioning to further reduces the required memory overhead when compared with HNSW and can effectively be run at the smallest possible RAM and heap sizes when HNSW would otherwise cause swapping and grind to a halt.  DiskBBQ largely scales linearlly with the total RAM.  And search performance is enhanced at scale as a subset of the total vector space is loaded.
 
 `m`
 :   (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types.
@@ -363,6 +364,12 @@ $$$dense-vector-index-options$$$
 `confidence_interval`
 :   (Optional, float) Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, this value restricts the values used when calculating the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation.
 
+`default_visit_percentage` {applies_to}`stack: ga 9.2`
+:   (Optional, integer) Only applicable to `bbq_disk`. Must be between 0 and 100.  0 will default to using `num_candidates` for calculating the percent visited. Increasing `default_visit_percentage` tends to improve the accuracy of the final results. Defaults to ~1% per shard for every 1 million vectors.
+
+`cluster_size` {applies_to}`stack: ga 9.2`
+:   (Optional, integer) Only applicable to `bbq_disk`.  The number of vectors per cluster.  Smaller cluster sizes increases accuracy at the cost of performance. Defaults to `384`. Must be a value between `64` and `65536`.
+
 `rescore_vector` {applies_to}`stack: preview 9.0, ga 9.1`
 :   (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
 :::::{dropdown} Properties of rescore_vector

docs/reference/elasticsearch/rest-apis/retrievers/knn-retriever.md

@@ -41,6 +41,12 @@ A kNN retriever returns top documents from a [k-nearest neighbor search (kNN)](d
     The number of nearest neighbor candidates to consider per shard. Needs to be greater than `k`, or `size` if `k` is omitted, and cannot exceed 10,000. {{es}} collects `num_candidates` results from each shard, then merges them to find the top `k` results. Increasing `num_candidates` tends to improve the accuracy of the final `k` results. Defaults to `Math.min(1.5 * k, 10_000)`.
 
 
+`visit_percentage` {applies_to}`stack: ga 9.2`
+:   (Optional, float)
+
+    The percentage of vectors to explore per shard while doing knn search with `bbq_disk`. Must be between 0 and 100.  0 will default to using `num_candidates` for calculating the percent visited. Increasing `visit_percentage` tends to improve the accuracy of the final results.  If `visit_percentage` is set for `bbq_disk`, `num_candidates` is ignored. Defaults to ~1% per shard for every 1 million vectors.
+
+
 `filter`
 :   (Optional, [query object or list of query objects](/reference/query-languages/querydsl.md))
 

docs/reference/query-languages/esql.md

@@ -1,4 +1,7 @@
 ---
+applies_to:
+  stack:
+  serverless:
 navigation_title: "{{esql}}"
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-language.html

docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md

@@ -1,5 +1,6 @@
 ```yaml {applies_to}
 stack: preview 9.0.0, ga 9.1.0
+serverless: ga
 ```
 
 `LOOKUP JOIN` enables you to add data from another index, AKA a 'lookup'