Merge branch 'main' into knowledge-base-index-privileges

Author: Supplementing

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy

@@ -46,8 +46,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
         javaResource("myserver", "transport/definitions/named/" + name + ".csv", ids)
     }
 
-    def initialTransportVersion(String name, String id) {
-        javaResource("myserver", "transport/definitions/initial/" + name + ".csv", id)
+    def unreferencedTransportVersion(String name, String id) {
+        javaResource("myserver", "transport/definitions/unreferenced/" + name + ".csv", id)
     }
 
     def definedAndUsedTransportVersion(String name, String ids) {
@@ -101,7 +101,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
         """
         namedTransportVersion("existing_91", "8012000")
         namedTransportVersion("existing_92", "8123000,8012001")
-        initialTransportVersion("initial_9_0_0", "8000000")
+        unreferencedTransportVersion("initial_9_0_0", "8000000")
         latestTransportVersion("9.2", "existing_92", "8123000")
         latestTransportVersion("9.1", "existing_92", "8012001")
         // a mock version of TransportVersion, just here so we can compile Dummy.java et al
@@ -303,4 +303,14 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
         assertDefinitionsFailure(result, "Transport version definition file " +
             "[myserver/src/main/resources/transport/definitions/named/patch.csv] has patch version 8015001 as primary id")
     }
+
+    def "unreferenced directory is optional"() {
+        given:
+        file("myserver/src/main/resources/transport/unreferenced/initial_9_0_0.csv").delete()
+        file("myserver/src/main/resources/transport/unreferenced").deleteDir()
+        when:
+        def result = gradleRunner(":myserver:validateTransportVersionDefinitions").build()
+        then:
+        result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -107,10 +107,13 @@ public void validateTransportVersions() throws IOException {
         // now load all definitions, do some validation and record them by various keys for later quick lookup
         // NOTE: this must run after loading referenced names and existing definitions
         // NOTE: this is sorted so that the order of cross validation is deterministic
-        for (String subDir : List.of("initial", "named")) {
-            try (var definitionsStream = Files.list(definitionsDir.resolve(subDir)).sorted()) {
-                for (var definitionFile : definitionsStream.toList()) {
-                    recordAndValidateDefinition(readDefinitionFile(definitionFile));
+        for (String subDirName : List.of("unreferenced", "named")) {
+            Path subDir = definitionsDir.resolve(subDirName);
+            if (Files.isDirectory(subDir)) {
+                try (var definitionsStream = Files.list(subDir).sorted()) {
+                    for (var definitionFile : definitionsStream.toList()) {
+                        recordAndValidateDefinition(readDefinitionFile(definitionFile));
+                    }
                 }
             }
         }

docs/changelog/131907.yaml

@@ -0,0 +1,26 @@
+pr: 131907
+summary: Enable `exclude_source_vectors` by default for new indices
+area: Vector Search
+type: breaking
+issues: []
+breaking:
+  title: Enable `exclude_source_vectors` by default for new indices
+  area: Search
+  details: |-
+    The `exclude_source_vectors` setting is now enabled by default for newly created indices.
+    This means that vector fields (e.g., `dense_vector`) are no longer stored in the `_source` field
+    by default, although they remain fully accessible through search and retrieval operations.
+
+    Instead of being persisted in `_source`, vectors are now rehydrated on demand from the underlying
+    index structures when needed. This reduces index size and improves performance for typical vector
+    search workloads where the original vector values do not need to be part of the `_source`.
+
+    If your use case requires vector fields to be stored in `_source`, you can disable this behavior by
+    setting `exclude_source_vectors: false` at index creation time.
+  impact: |-
+    Vector fields will no longer be stored in `_source` by default for new indices. Applications or tools
+    that expect to see vector fields in `_source` (for raw document inspection)
+    may need to be updated or configured to explicitly retain vectors using `exclude_source_vectors: false`.
+
+    Retrieval of vector fields via search or the `_source` API remains fully supported.
+  notable: true

docs/changelog/132766.yaml

@@ -0,0 +1,10 @@
+pr: 132766
+summary: Change `reporting_user` role to leverage reserved kibana privileges
+area: Authorization
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate the built-in `reporting_user` role.
+  area: Authorization
+  details: The `reporting_user` role is deprecated. Administrators should manage access to Kibana's reporting features via custom roles which grant the necessary privileges.
+  impact: This role will be removed in a future version. Administrators should migrate to custom roles to avoid interruption.

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -102,6 +102,81 @@ PUT my-index-2
 
 {{es}} uses the [HNSW algorithm](https://arxiv.org/abs/1603.09320) to support efficient kNN search. Like most kNN algorithms, HNSW is an approximate method that sacrifices result accuracy for improved speed.
 
+## Accessing `dense_vector` fields in search responses
+```{applies_to}
+stack: ga 9.2
+serverless: ga
+```
+
+By default, `dense_vector` fields are **not included in `_source`** in responses from the `_search`, `_msearch`, `_get`, and `_mget` APIs.
+This helps reduce response size and improve performance, especially in scenarios where vectors are used solely for similarity scoring and not required in the output.
+
+To retrieve vector values explicitly, you can use:
+
+* The `fields` option to request specific vector fields directly:
+
+```console
+POST my-index-2/_search
+{
+  "fields": ["my_vector"]
+}
+```
+
+- The `_source.exclude_vectors` flag to re-enable vector inclusion in `_source` responses:
+
+```console
+POST my-index-2/_search
+{
+  "_source": {
+    "exclude_vectors": false
+  }
+}
+```
+
+### Storage behavior and `_source`
+
+By default, `dense_vector` fields are **not stored in `_source`** on disk. This is also controlled by the index setting `index.mapping.exclude_source_vectors`.
+This setting is enabled by default for newly created indices and can only be set at index creation time.
+
+When enabled:
+
+* `dense_vector` fields are removed from `_source` and the rest of the `_source` is stored as usual.
+* If a request includes `_source` and vector values are needed (e.g., during recovery or reindex), the vectors are rehydrated from their internal format.
+
+This setting is compatible with synthetic `_source`, where the entire `_source` document is reconstructed from columnar storage. In full synthetic mode, no `_source` is stored on disk, and all fields — including vectors — are rebuilt when needed.
+
+### Rehydration and precision
+
+When vector values are rehydrated (e.g., for reindex, recovery, or explicit `_source` requests), they are restored from their internal format. Internally, vectors are stored at float precision, so if they were originally indexed as higher-precision types (e.g., `double` or `long`), the rehydrated values will have reduced precision. This lossy representation is intended to save space while preserving search quality.
+
+### Storing original vectors in `_source`
+
+If you want to preserve the original vector values exactly as they were provided, you can re-enable vector storage in `_source`:
+
+```console
+PUT my-index-include-vectors
+{
+  "settings": {
+    "index.mapping.exclude_source_vectors": false
+  },
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "dense_vector"
+      }
+    }
+  }
+}
+```
+
+When this setting is disabled:
+
+* `dense_vector` fields are stored as part of the `_source`, exactly as indexed.
+* The index will store both the original `_source` value and the internal representation used for vector search, resulting in increased storage usage.
+* Vectors are once again returned in `_source` by default in all relevant APIs, with no need to use `exclude_vectors` or `fields`.
+
+This configuration is appropriate when full source fidelity is required, such as for auditing or round-tripping exact input values.
+
 ## Automatically quantize vectors for kNN search [dense-vector-quantization]
 
 The `dense_vector` type supports quantization to reduce the memory footprint required when [searching](docs-content://solutions/search/vector/knn.md#approximate-knn) `float` vectors. The three following quantization strategies are supported:
@@ -266,16 +341,16 @@ $$$dense-vector-index-options$$$
 `type`
 :   (Required, string) The type of kNN algorithm to use. Can be either any of:
     * `hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) for scalable approximate kNN search. This supports all `element_type` values.
-    * `int8_hnsw` - The default index type for some float vectors: 
-        
-      * {applies_to}`stack: ga 9.1` Default for float vectors with less than 384 dimensions. 
+    * `int8_hnsw` - The default index type for some float vectors:
+
+      * {applies_to}`stack: ga 9.1` Default for float vectors with less than 384 dimensions.
       * {applies_to}`stack: ga 9.0` Default for float all vectors.
-      
+
       This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 4x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization).
     * `int4_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 8x at the cost of some accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization).
     * `bbq_hnsw` - This utilizes the [HNSW algorithm](https://arxiv.org/abs/1603.09320) in addition to automatically binary quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 32x at the cost of accuracy. See [Automatically quantize vectors for kNN search](#dense-vector-quantization).
-        
-      {applies_to}`stack: ga 9.1` `bbq_hnsw` is the default index type for float vectors with greater than or equal to 384 dimensions. 
+
+      {applies_to}`stack: ga 9.1` `bbq_hnsw` is the default index type for float vectors with greater than or equal to 384 dimensions.
     * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values.
     * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`.
     * `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports `element_type` of `float`.
@@ -295,8 +370,8 @@ $$$dense-vector-index-options$$$
 :   (Optional, object) An optional section that configures automatic vector rescoring on knn queries for the given field. Only applicable to quantized index types.
 :::::{dropdown} Properties of rescore_vector
 `oversample`
-:   (required, float) The amount to oversample the search results by. This value should be one of the following: 
-    * Greater than `1.0` and less than `10.0` 
+:   (required, float) The amount to oversample the search results by. This value should be one of the following:
+    * Greater than `1.0` and less than `10.0`
     * Exactly `0` to indicate no oversampling and rescoring should occur {applies_to}`stack: ga 9.1`
     :   The higher the value, the more vectors will be gathered and rescored with the raw values per shard.
     :   In case a knn query specifies a `rescore_vector` parameter, the query `rescore_vector` parameter will be used instead.

docs/reference/elasticsearch/mapping-reference/rank-vectors.md

@@ -108,11 +108,81 @@ $$$rank-vectors-element-type$$$
 `dims`
 :   (Optional, integer) Number of vector dimensions. Can’t exceed `4096`. If `dims` is not specified, it will be set to the length of the first vector added to the field.
 
+## Accessing `dense_vector` fields in search responses
+```{applies_to}
+stack: ga 9.2
+serverless: ga
+```
+
+By default, `dense_vector` fields are **not included in `_source`** in responses from the `_search`, `_msearch`, `_get`, and `_mget` APIs.
+This helps reduce response size and improve performance, especially in scenarios where vectors are used solely for similarity scoring and not required in the output.
+
+To retrieve vector values explicitly, you can use:
+
+* The `fields` option to request specific vector fields directly:
+
+```console
+POST my-index-2/_search
+{
+  "fields": ["my_vector"]
+}
+```
+
+- The `_source.exclude_vectors` flag to re-enable vector inclusion in `_source` responses:
+
+```console
+POST my-index-2/_search
+{
+  "_source": {
+    "exclude_vectors": false
+  }
+}
+```
+
+### Storage behavior and `_source`
+
+By default, `rank_vectors` fields are not stored in `_source` on disk. This is also controlled by the index setting `index.mapping.exclude_source_vectors`.
+This setting is enabled by default for newly created indices and can only be set at index creation time.
+
+When enabled:
+
+* `rank_vectors` fields are removed from `_source` and the rest of the `_source` is stored as usual.
+* If a request includes `_source` and vector values are needed (e.g., during recovery or reindex), the vectors are rehydrated from their internal format.
+
+This setting is compatible with synthetic `_source`, where the entire `_source` document is reconstructed from columnar storage. In full synthetic mode, no `_source` is stored on disk, and all fields — including vectors — are rebuilt when needed.
+
+### Rehydration and precision
+
+When vector values are rehydrated (e.g., for reindex, recovery, or explicit `_source` requests), they are restored from their internal format. Internally, vectors are stored at float precision, so if they were originally indexed as higher-precision types (e.g., `double` or `long`), the rehydrated values will have reduced precision. This lossy representation is intended to save space while preserving search quality.
+
+### Storing original vectors in `_source`
+
+If you want to preserve the original vector values exactly as they were provided, you can re-enable vector storage in `_source`:
+
+```console
+PUT my-index-include-vectors
+{
+  "settings": {
+    "index.mapping.exclude_source_vectors": false
+  },
+  "mappings": {
+    "properties": {
+      "my_vector": {
+        "type": "rank_vectors",
+        "dims": 128
+      }
+    }
+  }
+}
+```
 
-## Synthetic `_source` [rank-vectors-synthetic-source]
+When this setting is disabled:
 
-`rank_vectors` fields support [synthetic `_source`](mapping-source-field.md#synthetic-source) .
+* `rank_vectors` fields are stored as part of the `_source`, exactly as indexed.
+* The index will store both the original `_source` value and the internal representation used for vector search, resulting in increased storage usage.
+* Vectors are once again returned in `_source` by default in all relevant APIs, with no need to use `exclude_vectors` or `fields`.
 
+This configuration is appropriate when full source fidelity is required, such as for auditing or round-tripping exact input values.
 
 ## Scoring with rank vectors [rank-vectors-scoring]