Merge branch 'main' into lucene_snapshot

Author: cbuescher

docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc

@@ -89,6 +89,16 @@ PUT semantic-embeddings
 It will be used to generate the embeddings based on the input text.
 Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text.
 
+[NOTE]
+====
+If you're using web crawlers or connectors to generate indices, you have to
+<<indices-put-mapping,update the index mappings>> for these indices to
+include the `semantic_text` field. Once the mapping is updated, you'll need to run
+a full web crawl or a full connector sync. This ensures that all existing
+documents are reprocessed and updated with the new semantic embeddings,
+enabling semantic search on the updated data.
+====
+
 
 [discrete]
 [[semantic-text-load-data]]
@@ -118,6 +128,13 @@ Create the embeddings from the text by reindexing the data from the `test-data`
 The data in the `content` field will be reindexed into the `content` semantic text field of the destination index.
 The reindexed data will be processed by the {infer} endpoint associated with the `content` semantic text field.
 
+[NOTE]
+====
+This step uses the reindex API to simulate data ingestion. If you are working with data that has already been indexed,
+rather than using the test-data set, reindexing is required to ensure that the data is processed by the {infer} endpoint
+and the necessary embeddings are generated.
+====
+
 [source,console]
 ------------------------------------------------------------
 POST _reindex?wait_for_completion=false

libs/simdvec/build.gradle

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask
 
 apply plugin: 'elasticsearch.publish'
@@ -32,7 +33,7 @@ tasks.matching { it.name == "compileMain21Java" }.configureEach {
 }
 
 tasks.named('test').configure {
-  if (JavaVersion.current().majorVersion.toInteger() >= 21) {
+  if (BuildParams.getRuntimeJavaVersion().majorVersion.toInteger() >= 21) {
     jvmArgs '--add-modules=jdk.incubator.vector'
   }
 }

modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -48,7 +49,7 @@ protected void doExecute(Task task, MainRequest request, ActionListener<MainResp
                 nodeName,
                 IndexVersion.current().luceneVersion().toString(),
                 clusterState.getClusterName(),
-                clusterState.metadata().clusterUUID(),
+                clusterState.metadata().clusterUUIDCommitted() ? clusterState.metadata().clusterUUID() : Metadata.UNKNOWN_CLUSTER_UUID,
                 Build.current()
             )
         );

modules/rest-root/src/test/java/org/elasticsearch/rest/root/MainActionTests.java

@@ -9,13 +9,14 @@
 
 package org.elasticsearch.rest.root;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestStatus;
@@ -26,7 +27,7 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.util.Collections;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -39,7 +40,7 @@ public void testMainActionClusterAvailable() {
         final ClusterService clusterService = mock(ClusterService.class);
         final ClusterName clusterName = new ClusterName("elasticsearch");
         final Settings settings = Settings.builder().put("node.name", "my-node").build();
-        ClusterBlocks blocks;
+        final ClusterBlocks blocks;
         if (randomBoolean()) {
             if (randomBoolean()) {
                 blocks = ClusterBlocks.EMPTY_CLUSTER_BLOCK;
@@ -73,7 +74,12 @@ public void testMainActionClusterAvailable() {
                 )
                 .build();
         }
-        ClusterState state = ClusterState.builder(clusterName).blocks(blocks).build();
+        final Metadata.Builder metadata = new Metadata.Builder();
+        if (randomBoolean()) {
+            metadata.clusterUUID(randomUUID());
+            metadata.clusterUUIDCommitted(randomBoolean());
+        }
+        final ClusterState state = ClusterState.builder(clusterName).metadata(metadata).blocks(blocks).build();
         when(clusterService.state()).thenReturn(state);
 
         TransportService transportService = new TransportService(
@@ -85,21 +91,21 @@ public void testMainActionClusterAvailable() {
             null,
             Collections.emptySet()
         );
-        TransportMainAction action = new TransportMainAction(settings, transportService, mock(ActionFilters.class), clusterService);
-        AtomicReference<MainResponse> responseRef = new AtomicReference<>();
-        action.doExecute(mock(Task.class), new MainRequest(), new ActionListener<>() {
-            @Override
-            public void onResponse(MainResponse mainResponse) {
-                responseRef.set(mainResponse);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                logger.error("unexpected error", e);
-            }
-        });
+        final AtomicBoolean listenerCalled = new AtomicBoolean();
+        new TransportMainAction(settings, transportService, mock(ActionFilters.class), clusterService).doExecute(
+            mock(Task.class),
+            new MainRequest(),
+            ActionTestUtils.assertNoFailureListener(mainResponse -> {
+                assertNotNull(mainResponse);
+                assertEquals(
+                    state.metadata().clusterUUIDCommitted() ? state.metadata().clusterUUID() : Metadata.UNKNOWN_CLUSTER_UUID,
+                    mainResponse.getClusterUuid()
+                );
+                assertFalse(listenerCalled.getAndSet(true));
+            })
+        );
 
-        assertNotNull(responseRef.get());
+        assertTrue(listenerCalled.get());
         verify(clusterService, times(1)).state();
     }
 }

renovate.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "github>elastic/renovate-config:only-chainguard"
+  ],
+  "customManagers": [
+    {
+      "description": "Extract Wolfi images from elasticsearch DockerBase configuration",
+      "customType": "regex",
+      "fileMatch": [
+        "build\\-tools\\-internal\\/src\\/main\\/java\\/org\\/elasticsearch\\/gradle\\/internal\\/DockerBase\\.java$"
+      ],
+      "matchStrings": [
+        "\\s*\"?(?<depName>[^\\s:@\"]+)(?::(?<currentValue>[-a-zA-Z0-9.]+))?(?:@(?<currentDigest>sha256:[a-zA-Z0-9]+))?\"?"
+      ],
+      "currentValueTemplate": "{{#if currentValue}}{{{currentValue}}}{{else}}latest{{/if}}",
+      "autoReplaceStringTemplate": "\"{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"",
+      "datasourceTemplate": "docker"
+    }
+  ]
+}

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -236,6 +236,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0);
     public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0);
     public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0);
+    public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchService;
+import org.elasticsearch.search.SearchSortValues;
 import org.elasticsearch.search.aggregations.AggregationReduceContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -51,6 +52,7 @@
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rank.RankDoc;
 import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext;
+import org.elasticsearch.search.sort.ShardDocSortField;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
@@ -464,6 +466,13 @@ private static SearchHits getHits(
                     assert shardDoc instanceof RankDoc;
                     searchHit.setRank(((RankDoc) shardDoc).rank);
                     searchHit.score(shardDoc.score);
+                    long shardAndDoc = ShardDocSortField.encodeShardAndDoc(shardDoc.shardIndex, shardDoc.doc);
+                    searchHit.sortValues(
+                        new SearchSortValues(
+                            new Object[] { shardDoc.score, shardAndDoc },
+                            new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }
+                        )
+                    );
                 } else if (sortedTopDocs.isSortedByField) {
                     FieldDoc fieldDoc = (FieldDoc) shardDoc;
                     searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.sortValueFormats);

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -695,6 +695,11 @@ public long version() {
         return this.version;
     }
 
+    /**
+     * @return A UUID which identifies this cluster. Nodes record the UUID of the cluster they first join on disk, and will then refuse to
+     * join clusters with different UUIDs. Note that when the cluster is forming for the first time this value may not yet be committed,
+     * and therefore it may change. Check {@link #clusterUUIDCommitted()} to verify that the value is committed if needed.
+     */
     public String clusterUUID() {
         return this.clusterUUID;
     }

server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java

@@ -19,10 +19,10 @@
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
-import org.elasticsearch.script.field.vectors.ESVectorUtil;
 
 import java.io.IOException;
 
@@ -103,7 +103,7 @@ public RandomVectorScorer getRandomVectorScorer(
     }
 
     static float hammingScore(byte[] a, byte[] b) {
-        return ((a.length * Byte.SIZE) - ESVectorUtil.xorBitCount(a, b)) / (float) (a.length * Byte.SIZE);
+        return ((a.length * Byte.SIZE) - VectorUtil.xorBitCount(a, b)) / (float) (a.length * Byte.SIZE);
     }
 
     static class HammingVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer {

server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java

@@ -103,7 +103,7 @@ public double l1Norm(List<Number> queryVector) {
 
     @Override
     public int hamming(byte[] queryVector) {
-        return ESVectorUtil.xorBitCount(queryVector, vectorValue);
+        return VectorUtil.xorBitCount(queryVector, vectorValue);
     }
 
     @Override