Merge remote-tracking branch 'origin/synthetic_vectors' into synthetic_vectors

Author: jimczi

docs/changelog/127223.yaml

@@ -0,0 +1,5 @@
+pr: 127223
+summary: Wrap ES KNN queries with PatienceKNN query
+area: Vector Search
+type: feature
+issues: []

docs/changelog/130344.yaml

@@ -0,0 +1,5 @@
+pr: 130344
+summary: "Fix queries with missing index, `skip_unavailable` and filters"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/130382.yaml

@@ -0,0 +1,5 @@
+pr: 130382
+summary: Remove vectors from `_source` transparently
+area: "Mapping, Vector Search"
+type: enhancement
+issues: []

docs/reference/elasticsearch/index-settings/index-modules.md

@@ -259,3 +259,6 @@ $$$index-esql-stored-fields-sequential-proportion$$$
 
 `index.esql.stored_fields_sequential_proportion`
 :   Tuning parameter for deciding when {{esql}} will load [Stored fields](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#stored-fields) using a strategy tuned for loading dense sequence of documents. Allows values between 0.0 and 1.0 and defaults to 0.2. Indices with documents smaller than 10kb may see speed improvements loading `text` fields by setting this lower.
+
+$$$index-dense-vector-hnsw-early-termination$$$ `index.dense_vector.hnsw_early_termination`
+:   Whether to apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is only applicable to `dense_vector` fields with `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. Defaults to `false`.

docs/reference/query-languages/esql/_snippets/lists/aggregation-functions.md

@@ -1,13 +1,10 @@
 * [`AVG`](../../functions-operators/aggregation-functions.md#esql-avg)
-* [unavailable] [`AVG_OVER_TIME`](../../functions-operators/aggregation-functions.md#esql-avg_over_time)
 * [`COUNT`](../../functions-operators/aggregation-functions.md#esql-count)
 * [`COUNT_DISTINCT`](../../functions-operators/aggregation-functions.md#esql-count_distinct)
 * [`MAX`](../../functions-operators/aggregation-functions.md#esql-max)
-* [unavailable] [`MAX_OVER_TIME`](../../functions-operators/aggregation-functions.md#esql-max_over_time)
 * [`MEDIAN`](../../functions-operators/aggregation-functions.md#esql-median)
 * [`MEDIAN_ABSOLUTE_DEVIATION`](../../functions-operators/aggregation-functions.md#esql-median_absolute_deviation)
 * [`MIN`](../../functions-operators/aggregation-functions.md#esql-min)
-* [unavailable] [`MIN_OVER_TIME`](../../functions-operators/aggregation-functions.md#esql-min_over_time)
 * [`PERCENTILE`](../../functions-operators/aggregation-functions.md#esql-percentile)
 * [preview] [`ST_CENTROID_AGG`](../../functions-operators/aggregation-functions.md#esql-st_centroid_agg)
 * [preview] [`ST_EXTENT_AGG`](../../functions-operators/aggregation-functions.md#esql-st_extent_agg)
@@ -16,6 +13,3 @@
 * [`TOP`](../../functions-operators/aggregation-functions.md#esql-top)
 * [preview] [`VALUES`](../../functions-operators/aggregation-functions.md#esql-values)
 * [`WEIGHTED_AVG`](../../functions-operators/aggregation-functions.md#esql-weighted_avg)
-* [unavailable] [`FIRST_OVER_TIME`](../../functions-operators/aggregation-functions.md#esql-first_over_time)
-* [unavailable] [`LAST_OVER_TIME`](../../functions-operators/aggregation-functions.md#esql-last_over_time)
-* [unavailable] [`RATE`](../../functions-operators/aggregation-functions.md#esql-rate)

docs/reference/query-languages/esql/_snippets/lists/source-commands.md

@@ -1,4 +1,3 @@
 * [`FROM`](../../commands/source-commands.md#esql-from)
-* [`TS`](../../commands/source-commands.md#esql-ts)
 * [`ROW`](../../commands/source-commands.md#esql-row)
 * [`SHOW`](../../commands/source-commands.md#esql-show)

docs/reference/query-languages/esql/commands/source-commands.md

@@ -20,9 +20,6 @@ An {{esql}} source command produces a table, typically with data from {{es}}. An
 :::{include} ../_snippets/commands/layout/from.md
 :::
 
-:::{include} ../_snippets/commands/layout/ts.md
-:::
-
 :::{include} ../_snippets/commands/layout/row.md
 :::
 

docs/reference/query-languages/esql/functions-operators/aggregation-functions.md

@@ -15,9 +15,6 @@ The [`STATS`](/reference/query-languages/esql/commands/processing-commands.md#es
 :::{include} ../_snippets/functions/layout/avg.md
 :::
 
-:::{include} ../_snippets/functions/layout/avg_over_time.md
-:::
-
 :::{include} ../_snippets/functions/layout/count.md
 :::
 
@@ -27,9 +24,6 @@ The [`STATS`](/reference/query-languages/esql/commands/processing-commands.md#es
 :::{include} ../_snippets/functions/layout/max.md
 :::
 
-:::{include} ../_snippets/functions/layout/max_over_time.md
-:::
-
 :::{include} ../_snippets/functions/layout/median.md
 :::
 
@@ -39,9 +33,6 @@ The [`STATS`](/reference/query-languages/esql/commands/processing-commands.md#es
 :::{include} ../_snippets/functions/layout/min.md
 :::
 
-:::{include} ../_snippets/functions/layout/min_over_time.md
-:::
-
 :::{include} ../_snippets/functions/layout/percentile.md
 :::
 
@@ -65,12 +56,3 @@ The [`STATS`](/reference/query-languages/esql/commands/processing-commands.md#es
 
 :::{include} ../_snippets/functions/layout/weighted_avg.md
 :::
-
-:::{include} ../_snippets/functions/layout/first_over_time.md
-:::
-
-:::{include} ../_snippets/functions/layout/last_over_time.md
-:::
-
-:::{include} ../_snippets/functions/layout/rate.md
-:::

qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java

@@ -48,7 +48,8 @@ record CmdLineArgs(
     VectorSimilarityFunction vectorSpace,
     int quantizeBits,
     VectorEncoding vectorEncoding,
-    int dimensions
+    int dimensions,
+    boolean earlyTermination
 ) implements ToXContentObject {
 
     static final ParseField DOC_VECTORS_FIELD = new ParseField("doc_vectors");
@@ -71,6 +72,7 @@ record CmdLineArgs(
     static final ParseField QUANTIZE_BITS_FIELD = new ParseField("quantize_bits");
     static final ParseField VECTOR_ENCODING_FIELD = new ParseField("vector_encoding");
     static final ParseField DIMENSIONS_FIELD = new ParseField("dimensions");
+    static final ParseField EARLY_TERMINATION_FIELD = new ParseField("early_termination");
 
     static CmdLineArgs fromXContent(XContentParser parser) throws IOException {
         Builder builder = PARSER.apply(parser, null);
@@ -100,6 +102,7 @@ static CmdLineArgs fromXContent(XContentParser parser) throws IOException {
         PARSER.declareInt(Builder::setQuantizeBits, QUANTIZE_BITS_FIELD);
         PARSER.declareString(Builder::setVectorEncoding, VECTOR_ENCODING_FIELD);
         PARSER.declareInt(Builder::setDimensions, DIMENSIONS_FIELD);
+        PARSER.declareBoolean(Builder::setEarlyTermination, EARLY_TERMINATION_FIELD);
     }
 
     @Override
@@ -158,6 +161,7 @@ static class Builder {
         private int quantizeBits = 8;
         private VectorEncoding vectorEncoding = VectorEncoding.FLOAT32;
         private int dimensions;
+        private boolean earlyTermination;
 
         public Builder setDocVectors(String docVectors) {
             this.docVectors = PathUtils.get(docVectors);
@@ -259,6 +263,11 @@ public Builder setDimensions(int dimensions) {
             return this;
         }
 
+        public Builder setEarlyTermination(Boolean patience) {
+            this.earlyTermination = patience;
+            return this;
+        }
+
         public CmdLineArgs build() {
             if (docVectors == null) {
                 throw new IllegalArgumentException("Document vectors path must be provided");
@@ -288,7 +297,8 @@ public CmdLineArgs build() {
                 vectorSpace,
                 quantizeBits,
                 vectorEncoding,
-                dimensions
+                dimensions,
+                earlyTermination
             );
         }
     }

qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java

@@ -211,7 +211,7 @@ public static void main(String[] args) throws Exception {
                 for (int i = 0; i < results.length; i++) {
                     int nProbe = nProbes[i];
                     KnnSearcher knnSearcher = new KnnSearcher(indexPath, cmdLineArgs, nProbe);
-                    knnSearcher.runSearch(results[i]);
+                    knnSearcher.runSearch(results[i], cmdLineArgs.earlyTermination());
                 }
             }
             formattedResults.results.addAll(List.of(results));