Merge branch 'main' into unmute-test-87695d2

Author: pabloem

modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java

@@ -1359,7 +1359,7 @@ public void testKnnQueryNotSupportedInPercolator() throws IOException {
             """);
         indicesAdmin().prepareCreate("index1").setMapping(mappings).get();
         ensureGreen();
-        QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, null, null);
+        QueryBuilder knnVectorQueryBuilder = new KnnVectorQueryBuilder("my_vector", new float[] { 1, 1, 1, 1, 1 }, 10, 10, 10f, null, null);
 
         IndexRequestBuilder indexRequestBuilder = prepareIndex("index1").setId("knn_query1")
             .setSource(jsonBuilder().startObject().field("my_query", knnVectorQueryBuilder).endObject());

muted-tests.yml

@@ -528,6 +528,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {csv-spec:fork.ForkWithMixOfCommands}
   issue: https://github.com/elastic/elasticsearch/issues/134135
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
+  method: test {csv-spec:inlinestats.MultiIndexInlinestatsOfMultiTypedField}
+  issue: https://github.com/elastic/elasticsearch/issues/133973
 
 # Examples:
 #

qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchErrorTraceIT.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.ErrorTraceHelper;
@@ -25,19 +24,15 @@
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.xcontent.XContentType;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.Collection;
-import java.util.function.BooleanSupplier;
 
 import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
 
 public class SearchErrorTraceIT extends HttpSmokeTestCase {
-    private BooleanSupplier hasStackTrace;
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
@@ -49,18 +44,6 @@ public static void setDebugLogLevel() {
         Configurator.setLevel(SearchService.class, Level.DEBUG);
     }
 
-    @Before
-    public void setupMessageListener() {
-        hasStackTrace = ErrorTraceHelper.setupErrorTraceListener(internalCluster());
-        // TODO: make this test work with batched query execution by enhancing ErrorTraceHelper.setupErrorTraceListener
-        updateClusterSettings(Settings.builder().put(SearchService.BATCHED_QUERY_PHASE.getKey(), false));
-    }
-
-    @After
-    public void resetSettings() {
-        updateClusterSettings(Settings.builder().putNull(SearchService.BATCHED_QUERY_PHASE.getKey()));
-    }
-
     private void setupIndexWithDocs() {
         createIndex("test1", "test2");
         indexRandom(
@@ -86,7 +69,7 @@ public void testSearchFailingQueryErrorTraceDefault() throws IOException {
             }
             """);
         getRestClient().performRequest(searchRequest);
-        assertFalse(hasStackTrace.getAsBoolean());
+        ErrorTraceHelper.assertStackTraceCleared(internalCluster());
     }
 
     public void testSearchFailingQueryErrorTraceTrue() throws IOException {
@@ -105,7 +88,7 @@ public void testSearchFailingQueryErrorTraceTrue() throws IOException {
             """);
         searchRequest.addParameter("error_trace", "true");
         getRestClient().performRequest(searchRequest);
-        assertTrue(hasStackTrace.getAsBoolean());
+        ErrorTraceHelper.assertStackTraceObserved(internalCluster());
     }
 
     public void testSearchFailingQueryErrorTraceFalse() throws IOException {
@@ -124,7 +107,7 @@ public void testSearchFailingQueryErrorTraceFalse() throws IOException {
             """);
         searchRequest.addParameter("error_trace", "false");
         getRestClient().performRequest(searchRequest);
-        assertFalse(hasStackTrace.getAsBoolean());
+        ErrorTraceHelper.assertStackTraceCleared(internalCluster());
     }
 
     public void testDataNodeLogsStackTrace() throws IOException {
@@ -173,7 +156,7 @@ public void testMultiSearchFailingQueryErrorTraceDefault() throws IOException {
             new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null))
         );
         getRestClient().performRequest(searchRequest);
-        assertFalse(hasStackTrace.getAsBoolean());
+        ErrorTraceHelper.assertStackTraceCleared(internalCluster());
     }
 
     public void testMultiSearchFailingQueryErrorTraceTrue() throws IOException {
@@ -190,7 +173,7 @@ public void testMultiSearchFailingQueryErrorTraceTrue() throws IOException {
         );
         searchRequest.addParameter("error_trace", "true");
         getRestClient().performRequest(searchRequest);
-        assertTrue(hasStackTrace.getAsBoolean());
+        ErrorTraceHelper.assertStackTraceObserved(internalCluster());
     }
 
     public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException {
@@ -207,8 +190,7 @@ public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException {
         );
         searchRequest.addParameter("error_trace", "false");
         getRestClient().performRequest(searchRequest);
-
-        assertFalse(hasStackTrace.getAsBoolean());
+        ErrorTraceHelper.assertStackTraceCleared(internalCluster());
     }
 
     public void testDataNodeLogsStackTraceMultiSearch() throws IOException {

renovate.json

@@ -1,7 +1,8 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "github>elastic/renovate-config:only-chainguard",
+    "github>elastic/renovate-config:base",
+    "github>elastic/renovate-config:ci-agent-images",
     ":disableDependencyDashboard"
   ],
   "schedule": [
@@ -10,6 +11,7 @@
   "labels": [
     ">non-issue",
     ":Delivery/Packaging",
+    ":Delivery/Build",
     "Team:Delivery",
     "auto-merge-without-approval"
   ],
@@ -20,5 +22,21 @@
     "8.19",
     "8.18",
     "7.17"
+  ],
+  "packageRules": [
+    {
+      "description": "Don't manage dependencies by default",
+      "matchPackageNames": [
+        "*"
+      ],
+      "enabled": false
+    },
+    {
+      "description": "Explicitly enable Elastic Docker updates",
+      "matchPackageNames": [
+        "/docker.elastic.co/.*/"
+      ],
+      "enabled": true
+    }
   ]
 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.retrievers/20_knn_retriever.yml

@@ -53,6 +53,30 @@ setup:
   - match: {hits.hits.1._id: "3"}
   - match: {hits.hits.1.fields.name.0: "rabbit.jpg"}
 
+---
+"kNN retrieve with visit_percentage":
+  - requires:
+      cluster_features: "mapper.bbq_disk_support"
+      reason: 'bbq disk support required'
+  - do:
+      search:
+        index: index1
+        body:
+          fields: [ "name" ]
+          retriever:
+            knn:
+              field: vector
+              query_vector: [2, 2, 2, 2, 3]
+              k: 2
+              num_candidates: 3
+              visit_percentage: 1.0
+
+  - match: {hits.hits.0._id: "2"}
+  - match: {hits.hits.0.fields.name.0: "moose.jpg"}
+
+  - match: {hits.hits.1._id: "3"}
+  - match: {hits.hits.1.fields.name.0: "rabbit.jpg"}
+
 ---
 "kNN retriever with filter":
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/135_knn_query_nested_search_ivf.yml

@@ -191,3 +191,27 @@ setup:
   - match: {hits.hits.0._id: "3"}
   - match: {hits.hits.0.fields.name.0: "rabbit.jpg"}
   - match: { hits.hits.0.inner_hits.nested.hits.hits.0.fields.nested.0.paragraph_id.0: "0" }
+
+---
+"nested kNN search works with visit_percentage":
+  - do:
+      search:
+        index: test
+        body:
+          fields: [ "name" ]
+          query:
+            nested:
+              path: nested
+              query:
+                knn:
+                  field: nested.vector
+                  query_vector: [-0.5, 90, -10, 14.8, -156]
+                  num_candidates: 3
+                  visit_percentage: 1.0
+  - match: {hits.total.value: 3}
+
+  - match: {hits.hits.0._id: "2"}
+  - match: {hits.hits.0.fields.name.0: "moose.jpg"}
+
+  - match: {hits.hits.1._id: "3"}
+  - match: {hits.hits.1.fields.name.0: "rabbit.jpg"}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf.yml

@@ -106,6 +106,28 @@ setup:
   - match: { hits.hits.1._id: "3" }
   - match: { hits.hits.2._id: "2" }
 ---
+"Test knn search with visit_percentage":
+  - do:
+      search:
+        index: bbq_disk
+        body:
+          knn:
+            field: vector
+            query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                           0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                           0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                           -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                           -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                           -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                           -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                           -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+            k: 3
+            visit_percentage: 1.0
+
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.2._id: "2" }
+---
 "Vector rescoring has same scoring as exact search for kNN section":
   - skip:
       features: "headers"

server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java

@@ -142,7 +142,7 @@ public void testDenseVectorMappingUpdate() throws Exception {
         for (int i = 0; i < queryVector.length; i++) {
             queryVector[i] = randomFloatBetween(-1, 1, true);
         }
-        KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(VECTOR_FIELD, queryVector, null, null, null, null);
+        KnnVectorQueryBuilder queryBuilder = new KnnVectorQueryBuilder(VECTOR_FIELD, queryVector, null, null, null, null, null);
         assertNoFailuresAndResponse(
             client().prepareSearch(INDEX_NAME).setQuery(queryBuilder).setTrackTotalHits(true).setSize(expectedDocs),
             response -> {

server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java

@@ -124,7 +124,7 @@ public void testDirectIOUsed() {
             indexVectors();
 
             // do a search
-            var knn = List.of(new KnnSearchBuilder("fooVector", new VectorData(null, new byte[64]), 10, 20, null, null));
+            var knn = List.of(new KnnSearchBuilder("fooVector", new VectorData(null, new byte[64]), 10, 20, 10f, null, null));
             assertHitCount(prepareSearch("foo-vectors").setKnnSearch(knn), 10);
             mockLog.assertAllExpectationsMatched();
         }

server/src/internalClusterTest/java/org/elasticsearch/search/KnnSearchIT.java

@@ -77,29 +77,29 @@ public void testKnnSearchWithScroll() throws Exception {
         // test top level knn search
         {
             SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-            sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null)));
+            sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null)));
             executeScrollSearch(client, sourceBuilder, k);
         }
         // test top level knn search + another query
         {
             SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-            sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null)));
+            sourceBuilder.knnSearch(List.of(new KnnSearchBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null)));
             sourceBuilder.query(QueryBuilders.existsQuery("category").boost(10));
             executeScrollSearch(client, sourceBuilder, k + 10);
         }
 
         // test knn query
         {
             SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
-            sourceBuilder.query(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null));
+            sourceBuilder.query(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null));
             executeScrollSearch(client, sourceBuilder, k * numShards);
         }
         // test knn query + another query
         {
             SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
             sourceBuilder.query(
                 QueryBuilders.boolQuery()
-                    .should(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, null, null))
+                    .should(new KnnVectorQueryBuilder(VECTOR_FIELD, new float[] { 0, 0 }, k, 100, 10f, null, null))
                     .should(QueryBuilders.existsQuery("category").boost(10))
             );
             executeScrollSearch(client, sourceBuilder, k * numShards + 10);