Merge branch 'main' into markjhoy/add_sparse_vector_token_pruning_index_options

Author: markjhoy

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -126,7 +126,10 @@ public void scoreFromArray(Blackhole bh) throws IOException {
                 in.readFloats(corrections, 0, corrections.length);
                 int addition = Short.toUnsignedInt(in.readShort());
                 float score = scorer.score(
-                    result,
+                    result.lowerInterval(),
+                    result.upperInterval(),
+                    result.quantizedComponentSum(),
+                    result.additionalCorrection(),
                     VectorSimilarityFunction.EUCLIDEAN,
                     centroidDp,
                     corrections[0],
@@ -150,7 +153,10 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
                 in.readFloats(corrections, 0, corrections.length);
                 int addition = Short.toUnsignedInt(in.readShort());
                 float score = scorer.score(
-                    result,
+                    result.lowerInterval(),
+                    result.upperInterval(),
+                    result.quantizedComponentSum(),
+                    result.additionalCorrection(),
                     VectorSimilarityFunction.EUCLIDEAN,
                     centroidDp,
                     corrections[0],
@@ -175,7 +181,10 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
                     in.readFloats(corrections, 0, corrections.length);
                     int addition = Short.toUnsignedInt(in.readShort());
                     float score = scorer.score(
-                        result,
+                        result.lowerInterval(),
+                        result.upperInterval(),
+                        result.quantizedComponentSum(),
+                        result.additionalCorrection(),
                         VectorSimilarityFunction.EUCLIDEAN,
                         centroidDp,
                         corrections[0],
@@ -196,7 +205,16 @@ public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {
-                scorer.scoreBulk(binaryQueries[j], result, VectorSimilarityFunction.EUCLIDEAN, centroidDp, scratchScores);
+                scorer.scoreBulk(
+                    binaryQueries[j],
+                    result.lowerInterval(),
+                    result.upperInterval(),
+                    result.quantizedComponentSum(),
+                    result.additionalCorrection(),
+                    VectorSimilarityFunction.EUCLIDEAN,
+                    centroidDp,
+                    scratchScores
+                );
                 bh.consume(scratchScores);
             }
         }

docs/changelog/119967.yaml

@@ -0,0 +1,5 @@
+pr: 119967
+summary: Add `index_options` to `semantic_text` field mappings
+area: Mapping
+type: enhancement
+issues: [ ]

docs/changelog/129507.yaml

@@ -0,0 +1,6 @@
+pr: 129507
+summary: Using a temp `IndexService` for template validation
+area: Indices APIs
+type: bug
+issues:
+ - 129473

docs/changelog/129509.yaml

@@ -0,0 +1,6 @@
+pr: 129509
+summary: Fix NPE in `SemanticTextHighlighter`
+area: Search
+type: bug
+issues:
+ - 129501

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -28,7 +28,7 @@ service.
 
 Using `semantic_text`, you won’t need to specify how to generate embeddings for
 your data, or how to index it. The {{infer}} endpoint automatically determines
-the embedding generation, indexing, and query to use. 
+the embedding generation, indexing, and query to use.
 Newly created indices with `semantic_text` fields using dense embeddings will be
 [quantized](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization)
 to `bbq_hnsw` automatically.
@@ -111,6 +111,33 @@ the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/ope
 to create the endpoint. If not specified, the {{infer}} endpoint defined by
 `inference_id` will be used at both index and query time.
 
+`index_options`
+:   (Optional, string) Specifies the index options to override default values
+for the field. Currently, `dense_vector` index options are supported.
+For text embeddings, `index_options` may match any allowed
+[dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+
+An example of how to set index_options for a `semantic_text` field:
+
+```console
+PUT my-index-000004
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text",
+        "inference_id": "my-text-embedding-endpoint",
+        "index_options": {
+          "dense_vector": {
+            "type": "int4_flat"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
 `chunking_settings`
 :   (Optional, object) Settings for chunking text into smaller passages.
 If specified, these will override the chunking settings set in the {{infer-cap}}
@@ -138,8 +165,10 @@ To completely disable chunking, use the `none` chunking strategy.
     or `1`. Required for `sentence` type chunking settings
 
 ::::{warning}
-If the input exceeds the maximum token limit of the underlying model,  some services (such as OpenAI) may return an 
-error. In contrast, the `elastic` and `elasticsearch` services  will automatically truncate the input to fit within the 
+If the input exceeds the maximum token limit of the underlying model, some
+services (such as OpenAI) may return an
+error. In contrast, the `elastic` and `elasticsearch` services will
+automatically truncate the input to fit within the
 model's limit.
 ::::
 
@@ -173,7 +202,8 @@ For more details on chunking and how to configure chunking settings,
 see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference)
 in the Inference API documentation.
 
-You can pre-chunk the input by sending it to Elasticsearch as an array of strings.
+You can pre-chunk the input by sending it to Elasticsearch as an array of
+strings.
 Example:
 
 ```console
@@ -203,15 +233,20 @@ PUT test-index/_doc/1
 ```
 
 1. The text is pre-chunked and provided as an array of strings.
-   Each element in the array represents a single chunk that will be sent directly to the inference service without further chunking.
+   Each element in the array represents a single chunk that will be sent
+   directly to the inference service without further chunking.
 
 **Important considerations**:
 
-* When providing pre-chunked input, ensure that you set the chunking strategy to `none` to avoid additional processing.
-* Each chunk should be sized carefully, staying within the token limit of the inference service and the underlying model.
-* If a chunk exceeds the model's token limit, the behavior depends on the service:
-  * Some services (such as OpenAI) will return an error.
-  * Others (such as `elastic` and `elasticsearch`) will automatically truncate the input.
+* When providing pre-chunked input, ensure that you set the chunking strategy to
+  `none` to avoid additional processing.
+* Each chunk should be sized carefully, staying within the token limit of the
+  inference service and the underlying model.
+* If a chunk exceeds the model's token limit, the behavior depends on the
+  service:
+    * Some services (such as OpenAI) will return an error.
+    * Others (such as `elastic` and `elasticsearch`) will automatically truncate
+      the input.
 
 Refer
 to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)

docs/reference/elasticsearch/rest-apis/retrieve-selected-fields.md

@@ -17,7 +17,7 @@ By default, each hit in the search response includes the document [`_source`](/r
 You can use both of these methods, though the `fields` option is preferred because it consults both the document data and index mappings. In some instances, you might want to use [other methods](#field-retrieval-methods) of retrieving data.
 
 
-### The `fields` option [search-fields-param]
+## The `fields` option [search-fields-param]
 
 To retrieve specific fields in the search response, use the `fields` parameter. Because it consults the index mappings, the `fields` parameter provides several advantages over referencing the `_source` directly. Specifically, the `fields` parameter:
 
@@ -33,7 +33,7 @@ Other mapping options are also respected, including [`ignore_above`](/reference/
 The `fields` option returns values in the way that matches how {{es}} indexes them. For standard fields, this means that the `fields` option looks in `_source` to find the values, then parses and formats them using the mappings. Selected fields that can’t be found in `_source` are skipped.
 
 
-#### Retrieve specific fields [search-fields-request]
+### Retrieve specific fields [search-fields-request]
 
 The following search request uses the `fields` parameter to retrieve values for the `user.id` field, all fields starting with `http.response.`, and the `@timestamp` field.
 
@@ -69,7 +69,7 @@ By default, document metadata fields like `_id` or `_index` are not returned whe
 
 
 
-#### Response always returns an array [search-fields-response]
+### Response always returns an array [search-fields-response]
 
 The `fields` response always returns an array of values for each field, even when there is a single value in the `_source`. This is because {{es}} has no dedicated array type, and any field could contain multiple values. The `fields` parameter also does not guarantee that array values are returned in a specific order. See the mapping documentation on [arrays](/reference/elasticsearch/mapping-reference/array.md) for more background.
 
@@ -109,7 +109,7 @@ The response includes values as a flat list in the `fields` section for each hit
 ```
 
 
-#### Retrieve nested fields [search-fields-nested]
+### Retrieve nested fields [search-fields-nested]
 
 ::::{dropdown}
 The `fields` response for [`nested` fields](/reference/elasticsearch/mapping-reference/nested.md) is slightly different from that of regular object fields. While leaf values inside regular `object` fields are returned as a flat list, values inside `nested` fields are grouped to maintain the independence of each object inside the original nested array. For each entry inside a nested field array, values are again returned as a flat list unless there are other `nested` fields inside the parent nested object, in which case the same procedure is repeated again for the deeper nested fields.
@@ -246,7 +246,7 @@ However, when the `fields` pattern targets the nested `user` field directly, no
 
 
 
-#### Retrieve unmapped fields [retrieve-unmapped-fields]
+### Retrieve unmapped fields [retrieve-unmapped-fields]
 
 ::::{dropdown}
 By default, the `fields` parameter returns only values of mapped fields. However, {{es}} allows storing fields in `_source` that are unmapped, such as setting [dynamic field mapping](docs-content://manage-data/data-store/mapping/dynamic-field-mapping.md) to `false` or by using an object field with `enabled: false`. These options disable parsing and indexing of the object content.
@@ -326,7 +326,7 @@ The response will contain field results under the  `session_data.object.*` path,
 
 
 
-#### Ignored field values [ignored-field-values]
+### Ignored field values [ignored-field-values]
 
 ::::{dropdown}
 The `fields` section of the response only returns values that were valid when indexed. If your search request asks for values from a field that ignored certain values because they were malformed or too large these values are returned separately in an `ignored_field_values` section.
@@ -578,6 +578,7 @@ Also only leaf fields can be returned via the `stored_fields` option. If an obje
 On its own, `stored_fields` cannot be used to load fields in nested objects — if a field contains a nested object in its path, then no data will be returned for that stored field. To access nested fields, `stored_fields` must be used within an [`inner_hits`](/reference/elasticsearch/rest-apis/retrieve-inner-hits.md) block.
 ::::
 
+For an example that uses the `stored_fields` parameter, refer to [](retrieve-stored-fields.md). 
 
 
 ##### Disable stored fields [disable-stored-fields]

docs/reference/elasticsearch/rest-apis/retrieve-stored-fields.md

@@ -0,0 +1,84 @@
+---
+navigation_title: "Retrieve stored fields"
+mapped_pages:
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html
+applies_to:
+  stack: all
+---
+
+# Retrieve stored fields using the Get document API [get-stored-fields]
+
+Use the `stored_fields` query parameter in a [Get document](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get) API request to retrieve fields marked as stored (`"store": true`) in the index mapping.
+
+Fields not marked as stored are excluded from the response, even if specified in the request.
+
+::::{tip}
+In most cases, the [`fields`](retrieve-selected-fields.md#search-fields-param) and [`_source`](retrieve-selected-fields.md#source-filtering) parameters produce better results than `stored_fields`.
+::::
+
+For example, these PUT requests define a stored field in the mapping and add a document:
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "counter": {
+        "type": "integer",
+        "store": false
+      },
+      "tags": {
+        "type": "keyword",
+        "store": true
+      }
+    }
+  }
+}
+```
+
+```console
+PUT my-index-000001/_doc/1
+{
+  "counter": 1,
+  "tags": [ "production" ]
+}
+```
+
+% TEST[continued]
+
+This request retrieves the stored fields from the document:
+
+```console
+GET my-index-000001/_doc/1?stored_fields=tags,counter
+```
+
+% TEST[continued]
+
+The API returns the following response:
+
+```console-result
+{
+  "_index": "my-index-000001",
+  "_id": "1",
+  "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
+  "found": true,
+  "fields": {
+    "tags": [
+      "production"
+    ]
+  }
+}
+```
+
+% TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
+
+Although the `counter` field is specified in the request, it's not included in the response because it's not actually a stored field.
+
+Field values are returned as an array.
+
+::::{note}
+Only leaf fields can be retrieved with the `stored_fields` parameter. If you specify an object field instead, an error is returned.
+::::
+

docs/reference/elasticsearch/toc.yml

@@ -93,6 +93,7 @@ toc:
             - file: rest-apis/reindex-indices.md
             - file: rest-apis/retrieve-inner-hits.md
             - file: rest-apis/retrieve-selected-fields.md
+            - file: rest-apis/retrieve-stored-fields.md
             - file: rest-apis/retrievers.md
             - file: rest-apis/search-multiple-data-streams-indices.md
             - file: rest-apis/search-profile.md

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ES91OSQVectorsScorer.java

@@ -95,7 +95,10 @@ public void quantizeScoreBulk(byte[] q, int count, float[] scores) throws IOExce
      * Computes the score by applying the necessary corrections to the provided quantized distance.
      */
     public float score(
-        OptimizedScalarQuantizer.QuantizationResult queryCorrections,
+        float queryLowerInterval,
+        float queryUpperInterval,
+        int queryComponentSum,
+        float queryAdditionalCorrection,
         VectorSimilarityFunction similarityFunction,
         float centroidDp,
         float lowerInterval,
@@ -107,19 +110,19 @@ public float score(
         float ax = lowerInterval;
         // Here we assume `lx` is simply bit vectors, so the scaling isn't necessary
         float lx = upperInterval - ax;
-        float ay = queryCorrections.lowerInterval();
-        float ly = (queryCorrections.upperInterval() - ay) * FOUR_BIT_SCALE;
-        float y1 = queryCorrections.quantizedComponentSum();
+        float ay = queryLowerInterval;
+        float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE;
+        float y1 = queryComponentSum;
         float score = ax * ay * dimensions + ay * lx * (float) targetComponentSum + ax * ly * y1 + lx * ly * qcDist;
         // For euclidean, we need to invert the score and apply the additional correction, which is
         // assumed to be the squared l2norm of the centroid centered vectors.
         if (similarityFunction == EUCLIDEAN) {
-            score = queryCorrections.additionalCorrection() + additionalCorrection - 2 * score;
+            score = queryAdditionalCorrection + additionalCorrection - 2 * score;
             return Math.max(1 / (1f + score), 0);
         } else {
             // For cosine and max inner product, we need to apply the additional correction, which is
             // assumed to be the non-centered dot-product between the vector and the centroid
-            score += queryCorrections.additionalCorrection() + additionalCorrection - centroidDp;
+            score += queryAdditionalCorrection + additionalCorrection - centroidDp;
             if (similarityFunction == MAXIMUM_INNER_PRODUCT) {
                 return VectorUtil.scaleMaxInnerProductScore(score);
             }
@@ -140,7 +143,10 @@ public float score(
      */
     public void scoreBulk(
         byte[] q,
-        OptimizedScalarQuantizer.QuantizationResult queryCorrections,
+        float queryLowerInterval,
+        float queryUpperInterval,
+        int queryComponentSum,
+        float queryAdditionalCorrection,
         VectorSimilarityFunction similarityFunction,
         float centroidDp,
         float[] scores
@@ -154,7 +160,10 @@ public void scoreBulk(
         in.readFloats(additionalCorrections, 0, BULK_SIZE);
         for (int i = 0; i < BULK_SIZE; i++) {
             scores[i] = score(
-                queryCorrections,
+                queryLowerInterval,
+                queryUpperInterval,
+                queryComponentSum,
+                queryAdditionalCorrection,
                 similarityFunction,
                 centroidDp,
                 lowerIntervals[i],