Merge branch 'main' into fix_plugin_examples

Author: mosche

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -2,6 +2,9 @@
 navigation_title: "Semantic text"
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/semantic-text.html
+applies_to:
+  stack: ga 9.0
+  serverless: ga
 ---
 
 # Semantic text field type [semantic-text]
@@ -29,7 +32,8 @@ service.
 Using `semantic_text`, you won’t need to specify how to generate embeddings for
 your data, or how to index it. The {{infer}} endpoint automatically determines
 the embedding generation, indexing, and query to use.
-Newly created indices with `semantic_text` fields using dense embeddings will be
+
+{applies_to}`stack: ga 9.1`  Newly created indices with `semantic_text` fields using dense embeddings will be
 [quantized](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization)
 to `bbq_hnsw` automatically.
 
@@ -182,6 +186,15 @@ For more details on chunking and how to configure chunking settings,
 see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference)
 in the Inference API documentation.
 
+Refer
+to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)
+to learn more about semantic search using `semantic_text`.
+
+### Pre-chunking [pre-chunking]
+```{applies_to}
+stack: ga 9.1
+```
+
 You can pre-chunk the input by sending it to Elasticsearch as an array of
 strings.
 Example:
@@ -228,10 +241,6 @@ PUT test-index/_doc/1
     * Others (such as `elastic` and `elasticsearch`) will automatically truncate
       the input.
 
-Refer
-to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)
-to learn more about semantic search using `semantic_text`.
-
 ## Extracting relevant fragments from semantic text [semantic-text-highlighting]
 
 You can extract the most relevant fragments from a semantic text field by using
@@ -295,6 +304,11 @@ specified. It enables you to quickstart your semantic search by providing
 automatic {{infer}} and a dedicated query so you don’t need to provide further
 details.
 
+### Customizing using `semantic_text` parameters [custom-by-parameters]
+```{applies_to}
+stack: ga 9.1
+```
+
 If you want to override those defaults and customize the embeddings that
 `semantic_text` indexes, you can do so by
 modifying [parameters](#semantic-text-params):
@@ -328,6 +342,24 @@ PUT my-index-000004
 }
 ```
 
+### Customizing using ingest pipelines [custom-by-pipelines]
+```{applies_to}
+stack: ga 9.0
+```
+
+In case you want to customize data indexing, use the
+[`sparse_vector`](/reference/elasticsearch/mapping-reference/sparse-vector.md)
+or [`dense_vector`](/reference/elasticsearch/mapping-reference/dense-vector.md)
+field types and create an ingest pipeline with an 
+[{{infer}} processor](/reference/enrich-processor/inference-processor.md) to
+generate the embeddings.
+[This tutorial](docs-content://solutions/search/semantic-search/semantic-search-inference.md)
+walks you through the process. In these cases - when you use `sparse_vector` or
+`dense_vector` field types instead of the `semantic_text` field type to
+customize indexing - using the
+[`semantic_query`](/reference/query-languages/query-dsl/query-dsl-semantic-query.md)
+is not supported for querying the field data.
+
 ## Updates to `semantic_text` fields [update-script]
 
 For indices containing `semantic_text` fields, updates that use scripts have the

docs/reference/query-languages/query-dsl/query-dsl-semantic-query.md

@@ -2,6 +2,9 @@
 navigation_title: "Semantic"
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-semantic-query.html
+applies_to:
+  stack: ga 9.0
+  serverless: ga
 ---
 
 # Semantic query [query-dsl-semantic-query]

muted-tests.yml

@@ -512,6 +512,12 @@ tests:
 - class: org.elasticsearch.xpack.stack.StackYamlIT
   method: test {yaml=stack/10_basic/Test wrong data_stream type - logs from 9.2.0}
   issue: https://github.com/elastic/elasticsearch/issues/131803
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test151MachineDependentHeapWithSizeOverride
+  issue: https://github.com/elastic/elasticsearch/issues/123437
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests
+  method: testUpdates
+  issue: https://github.com/elastic/elasticsearch/issues/131795
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java

@@ -548,11 +548,7 @@ public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset) throw
             // Doubles from doc values ensures that the values are in order
             try (BlockLoader.FloatBuilder builder = factory.denseVectors(docs.count() - offset, dimensions)) {
                 for (int i = offset; i < docs.count(); i++) {
-                    int doc = docs.get(i);
-                    if (doc < iterator.docID()) {
-                        throw new IllegalStateException("docs within same block must be in order");
-                    }
-                    read(doc, builder);
+                    read(docs.get(i), builder);
                 }
                 return builder.build();
             }
@@ -564,7 +560,9 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build
         }
 
         private void read(int doc, BlockLoader.FloatBuilder builder) throws IOException {
-            if (iterator.advance(doc) == doc) {
+            if (iterator.docID() > doc) {
+                builder.appendNull();
+            } else if (iterator.docID() == doc || iterator.advance(doc) == doc) {
                 builder.beginPositionEntry();
                 float[] floats = floatVectorValues.vectorValue(iterator.index());
                 assert floats.length == dimensions

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java

@@ -94,10 +94,14 @@ public void testRetrieveTopNDenseVectorFieldData() {
                 var values = valuesList.get(id);
                 assertEquals(id, values.get(0));
                 List<Float> vectors = (List<Float>) values.get(1);
-                assertNotNull(vectors);
-                assertEquals(vector.size(), vectors.size());
-                for (int i = 0; i < vector.size(); i++) {
-                    assertEquals(vector.get(i), vectors.get(i), 0F);
+                if (vector == null) {
+                    assertNull(vectors);
+                } else {
+                    assertNotNull(vectors);
+                    assertEquals(vector.size(), vectors.size());
+                    for (int i = 0; i < vector.size(); i++) {
+                        assertEquals(vector.get(i), vectors.get(i), 0F);
+                    }
                 }
             });
         }
@@ -117,12 +121,18 @@ public void testRetrieveDenseVectorFieldData() {
                 ;
                 assertEquals(2, value.size());
                 Integer id = (Integer) value.get(0);
-                List<Float> vector = (List<Float>) value.get(1);
-                assertNotNull(vector);
                 List<Float> expectedVector = indexedVectors.get(id);
-                assertNotNull(expectedVector);
-                for (int i = 0; i < vector.size(); i++) {
-                    assertEquals(expectedVector.get(i), vector.get(i), 0F);
+                List<Float> vector = (List<Float>) value.get(1);
+                if (expectedVector == null) {
+                    assertNull(vector);
+                } else {
+                    assertNotNull(vector);
+                    assertEquals(expectedVector.size(), vector.size());
+                    assertNotNull(vector);
+                    assertNotNull(expectedVector);
+                    for (int i = 0; i < vector.size(); i++) {
+                        assertEquals(expectedVector.get(i), vector.get(i), 0F);
+                    }
                 }
             });
         }
@@ -168,11 +178,16 @@ public void setup() throws IOException {
         IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs; i++) {
             List<Float> vector = new ArrayList<>(numDims);
-            for (int j = 0; j < numDims; j++) {
-                vector.add(randomFloat());
+            if (rarely()) {
+                docs[i] = prepareIndex("test").setId("" + i).setSource("id", String.valueOf(i));
+                indexedVectors.put(i, null);
+            } else {
+                for (int j = 0; j < numDims; j++) {
+                    vector.add(randomFloat());
+                }
+                docs[i] = prepareIndex("test").setId("" + i).setSource("id", String.valueOf(i), "vector", vector);
+                indexedVectors.put(i, vector);
             }
-            docs[i] = prepareIndex("test").setId("" + i).setSource("id", String.valueOf(i), "vector", vector);
-            indexedVectors.put(i, vector);
         }
 
         indexRandom(true, docs);