Merge branch 'main' into native_vector_ops

Author: ChrisHegarty

docs/changelog/130531.yaml

@@ -0,0 +1,6 @@
+pr: 130531
+summary: Adding check for `isIndexed` in text fields when generating field exists
+  queries to avoid ISE when field is stored but not indexed or with `doc_values`
+area: Analysis
+type: bug
+issues: []

muted-tests.yml

@@ -290,9 +290,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test023InstallPluginUsingConfigFile
   issue: https://github.com/elastic/elasticsearch/issues/126145
-- class: org.elasticsearch.search.SearchWithRejectionsIT
-  method: testOpenContextsAfterRejections
-  issue: https://github.com/elastic/elasticsearch/issues/126340
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
   issue: https://github.com/elastic/elasticsearch/issues/123200
@@ -419,9 +416,6 @@ tests:
 - class: org.elasticsearch.packaging.test.TemporaryDirectoryConfigTests
   method: test21AcceptsCustomPathInDocker
   issue: https://github.com/elastic/elasticsearch/issues/128114
-- class: org.elasticsearch.xpack.ml.integration.InferenceIngestIT
-  method: testPipelineIngestWithModelAliases
-  issue: https://github.com/elastic/elasticsearch/issues/128417
 - class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
   method: testCCSClusterDetailsWhereAllShardsSkippedInCanMatch
   issue: https://github.com/elastic/elasticsearch/issues/128418
@@ -449,9 +443,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test150MachineDependentHeap
   issue: https://github.com/elastic/elasticsearch/issues/128120
-- class: org.elasticsearch.xpack.inference.InferenceGetServicesIT
-  method: testGetServicesWithCompletionTaskType
-  issue: https://github.com/elastic/elasticsearch/issues/128952
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test073RunEsAsDifferentUserAndGroupWithoutBindMounting
   issue: https://github.com/elastic/elasticsearch/issues/128996
@@ -528,12 +519,6 @@ tests:
   method: "builds distribution from branches via archives extractedAssemble [bwcDistVersion: 8.2.1, bwcProject: bugfix, expectedAssembleTaskName:
     extractedAssemble, #2]"
   issue: https://github.com/elastic/elasticsearch/issues/119871
-- class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
-  method: testRerank
-  issue: https://github.com/elastic/elasticsearch/issues/130009
-- class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
-  method: testCohereEmbeddings
-  issue: https://github.com/elastic/elasticsearch/issues/130010
 - class: geoip.GeoIpMultiProjectIT
   issue: https://github.com/elastic/elasticsearch/issues/130073
 - class: org.elasticsearch.xpack.esql.action.EnrichIT
@@ -545,9 +530,6 @@ tests:
 - class: org.elasticsearch.compute.aggregation.TopIntAggregatorFunctionTests
   method: testManyInitialManyPartialFinalRunnerThrowing
   issue: https://github.com/elastic/elasticsearch/issues/130145
-- class: org.elasticsearch.index.codec.vectors.cluster.KMeansLocalTests
-  method: testKMeansNeighbors
-  issue: https://github.com/elastic/elasticsearch/issues/130258
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=esql/10_basic/basic with documents_found}
   issue: https://github.com/elastic/elasticsearch/issues/130256
@@ -575,30 +557,18 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=msearch/20_typed_keys/Multisearch test with typed_keys parameter for sampler and significant terms}
   issue: https://github.com/elastic/elasticsearch/issues/130472
-- class: org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeansTests
-  method: testHKmeans
-  issue: https://github.com/elastic/elasticsearch/issues/130497
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   method: testProjectWhere
   issue: https://github.com/elastic/elasticsearch/issues/130504
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   method: testTopNPushedToLucene
   issue: https://github.com/elastic/elasticsearch/issues/130505
-- class: org.elasticsearch.xpack.logsdb.LogsdbTestSuiteIT
-  method: test {yaml=/52_esql_insist_operator_synthetic_source/FROM with INSIST_🐔and LIMIT 1}
-  issue: https://github.com/elastic/elasticsearch/issues/130506
-- class: org.elasticsearch.xpack.logsdb.LogsdbTestSuiteIT
-  method: test {yaml=/52_esql_insist_operator_synthetic_source/FROM with INSIST_🐔}
-  issue: https://github.com/elastic/elasticsearch/issues/130507
 - class: org.elasticsearch.common.ssl.DefaultJdkTrustConfigTests
   method: testGetSystemTrustStoreWithNoSystemProperties
   issue: https://github.com/elastic/elasticsearch/issues/130517
 - class: org.elasticsearch.common.ssl.DefaultJdkTrustConfigTests
   method: testGetNonPKCS11TrustStoreWithPasswordSet
   issue: https://github.com/elastic/elasticsearch/issues/130519
-- class: org.elasticsearch.repositories.blobstore.BlobStoreCorruptionIT
-  method: testCorruptionDetection
-  issue: https://github.com/elastic/elasticsearch/issues/130536
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=indices.resolve_index/10_basic_resolve_index/Resolve index with hidden and closed indices}
   issue: https://github.com/elastic/elasticsearch/issues/130568
@@ -632,6 +602,15 @@ tests:
 - class: org.elasticsearch.xpack.monitoring.exporter.http.HttpExporterTests
   method: testCreateRestClient
   issue: https://github.com/elastic/elasticsearch/issues/130600
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/130067
+- class: org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeansTests
+  method: testHKmeans
+  issue: https://github.com/elastic/elasticsearch/issues/130497
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  method: test {p0=search.vectors/40_knn_search/Dimensions are dynamically set}
+  issue: https://github.com/elastic/elasticsearch/issues/130626
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/160_exists_query.yml

@@ -45,6 +45,10 @@ setup:
                       type: keyword
                 text:
                   type: text
+                text_stored_not_indexed:
+                  type: text
+                  store: true
+                  index: false
 
   - do:
       headers:
@@ -70,6 +74,7 @@ setup:
               inner1: "foo"
               inner2: "bar"
             text: "foo bar"
+            text_stored_not_indexed: "foo bar"
 
   - do:
       headers:
@@ -94,6 +99,7 @@ setup:
             object:
               inner1: "foo"
             text: "foo bar"
+            text_stored_not_indexed: "foo bar"
 
   - do:
       headers:
@@ -119,6 +125,7 @@ setup:
             object:
               inner2: "bar"
             text: "foo bar"
+            text_stored_not_indexed: "foo bar"
 
   - do:
       index:
@@ -184,6 +191,12 @@ setup:
                       doc_values: false
                 text:
                   type: text
+                keyword_stored_norms_not_indexed:
+                  type: keyword
+                  doc_values: false
+                  index: false
+                  store: true
+                  norms: true
 
   - do:
       headers:
@@ -209,6 +222,7 @@ setup:
               inner1: "foo"
               inner2: "bar"
             text: "foo bar"
+            keyword_stored_norms_not_indexed: "foo bar"
 
   - do:
       headers:
@@ -233,6 +247,7 @@ setup:
             object:
               inner1: "foo"
             text: "foo bar"
+            keyword_stored_norms_not_indexed: "foo bar"
 
   - do:
       headers:
@@ -258,6 +273,7 @@ setup:
             object:
               inner2: "bar"
             text: "foo bar"
+            keyword_stored_norms_not_indexed: "foo bar"
 
   - do:
       index:
@@ -1268,3 +1284,48 @@ setup:
                 field: text
 
   - match: {hits.total: 1}
+
+---
+"Test exists query on text field with no dv, that is stored but not indexed":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ field_exists_query_for_text_fields_no_index_or_dv ]
+      test_runner_features: capabilities
+      reason: "Before the fix, this query would throw an ISE because the field is not indexed and has no doc values."
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          query:
+            exists:
+              field: text_stored_not_indexed
+
+  # this should not throw, but rather return 0 hits, as the field is not indexed nor it has doc values
+  - match: {hits.total: 0}
+
+
+---
+"Test exists query on keyword field with no dv, that is stored, with norms, but not indexed":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ field_exists_query_for_text_fields_no_index_or_dv ]
+      test_runner_features: capabilities
+      reason: "Before the fix, this query would throw an ISE because the field is not indexed and has no doc values."
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-no-dv
+        body:
+          query:
+            exists:
+              field: keyword_stored_norms_not_indexed
+
+  # this should not throw, but rather return 0 hits, as the field is not indexed nor it has doc values
+  - match: {hits.total: 0}

server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java

@@ -90,7 +90,7 @@ public void testCorruptionDetection() throws Exception {
                             "fallback message",
                             "org.elasticsearch.repositories.blobstore.BlobStoreRepository",
                             Level.ERROR,
-                            "index [*] shard generation [*] in ["
+                            "index [*] shard generation [*] in [default/"
                                 + repositoryName
                                 + "][*] not found - falling back to reading all shard snapshots"
                         )
@@ -100,7 +100,9 @@ public void testCorruptionDetection() throws Exception {
                             "shard blobs list",
                             "org.elasticsearch.repositories.blobstore.BlobStoreRepository",
                             Level.ERROR,
-                            "read shard snapshots [*] due to missing shard generation [*] for index [*] in [" + repositoryName + "][*]"
+                            "read shard snapshots [*] due to missing shard generation [*] for index [*] in [default/"
+                                + repositoryName
+                                + "][*]"
                         )
                     );
                     client().admin()

server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithRejectionsIT.java

@@ -58,7 +58,7 @@ public void testOpenContextsAfterRejections() throws Exception {
         }
         assertBusy(
             () -> assertThat(indicesAdmin().prepareStats().get().getTotal().getSearch().getOpenContexts(), equalTo(0L)),
-            1,
+            2,
             TimeUnit.SECONDS
         );
     }

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/HierarchicalKMeans.java

@@ -68,8 +68,8 @@ public KMeansResult cluster(FloatVectorValues vectors, int targetSize) throws IO
         KMeansIntermediate kMeansIntermediate = clusterAndSplit(vectors, targetSize);
         if (kMeansIntermediate.centroids().length > 1 && kMeansIntermediate.centroids().length < vectors.size()) {
             int localSampleSize = Math.min(kMeansIntermediate.centroids().length * samplesPerCluster / 2, vectors.size());
-            KMeansLocal kMeansLocal = new KMeansLocal(localSampleSize, maxIterations, clustersPerNeighborhood, DEFAULT_SOAR_LAMBDA);
-            kMeansLocal.cluster(vectors, kMeansIntermediate, true);
+            KMeansLocal kMeansLocal = new KMeansLocal(localSampleSize, maxIterations);
+            kMeansLocal.cluster(vectors, kMeansIntermediate, clustersPerNeighborhood, DEFAULT_SOAR_LAMBDA);
         }
 
         return kMeansIntermediate;

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/KMeansLocal.java

@@ -35,18 +35,10 @@ class KMeansLocal {
 
     final int sampleSize;
     final int maxIterations;
-    final int clustersPerNeighborhood;
-    final float soarLambda;
 
-    KMeansLocal(int sampleSize, int maxIterations, int clustersPerNeighborhood, float soarLambda) {
+    KMeansLocal(int sampleSize, int maxIterations) {
         this.sampleSize = sampleSize;
         this.maxIterations = maxIterations;
-        this.clustersPerNeighborhood = clustersPerNeighborhood;
-        this.soarLambda = soarLambda;
-    }
-
-    KMeansLocal(int sampleSize, int maxIterations) {
-        this(sampleSize, maxIterations, -1, -1f);
     }
 
     /**
@@ -198,8 +190,13 @@ private void computeNeighborhoods(float[][] centers, List<NeighborHood> neighbor
         }
     }
 
-    private int[] assignSpilled(FloatVectorValues vectors, List<NeighborHood> neighborhoods, float[][] centroids, int[] assignments)
-        throws IOException {
+    private int[] assignSpilled(
+        FloatVectorValues vectors,
+        List<NeighborHood> neighborhoods,
+        float[][] centroids,
+        int[] assignments,
+        float soarLambda
+    ) throws IOException {
         // SOAR uses an adjusted distance for assigning spilled documents which is
         // given by:
         //
@@ -264,6 +261,10 @@ private int[] assignSpilled(FloatVectorValues vectors, List<NeighborHood> neighb
         return spilledAssignments;
     }
 
+    record NeighborHood(int[] neighbors, float maxIntraDistance) {
+        static final NeighborHood EMPTY = new NeighborHood(new int[0], Float.POSITIVE_INFINITY);
+    }
+
     /**
      * cluster using a lloyd k-means algorithm that is not neighbor aware
      *
@@ -274,11 +275,7 @@ private int[] assignSpilled(FloatVectorValues vectors, List<NeighborHood> neighb
      * @throws IOException is thrown if vectors is inaccessible
      */
     void cluster(FloatVectorValues vectors, KMeansIntermediate kMeansIntermediate) throws IOException {
-        cluster(vectors, kMeansIntermediate, false);
-    }
-
-    record NeighborHood(int[] neighbors, float maxIntraDistance) {
-        static final NeighborHood EMPTY = new NeighborHood(new int[0], Float.POSITIVE_INFINITY);
+        doCluster(vectors, kMeansIntermediate, -1, -1);
     }
 
     /**
@@ -290,12 +287,23 @@ record NeighborHood(int[] neighbors, float maxIntraDistance) {
      *                     the prior assignments of the given vectors; care should be taken in
      *                     passing in a valid output object with a centroids array that is the size of centroids expected
      *                     and assignments that are the same size as the vectors.  The SOAR assignments are overwritten by this operation.
-     * @param neighborAware whether nearby neighboring centroids and their vectors should be used to update the centroid positions,
-     *                      implies SOAR assignments
-     * @throws IOException is thrown if vectors is inaccessible
+     * @param clustersPerNeighborhood number of nearby neighboring centroids to be used to update the centroid positions.
+     * @param soarLambda   lambda used for SOAR assignments
+     *
+     * @throws IOException is thrown if vectors is inaccessible or if the clustersPerNeighborhood is less than 2
      */
-    void cluster(FloatVectorValues vectors, KMeansIntermediate kMeansIntermediate, boolean neighborAware) throws IOException {
+    void cluster(FloatVectorValues vectors, KMeansIntermediate kMeansIntermediate, int clustersPerNeighborhood, float soarLambda)
+        throws IOException {
+        if (clustersPerNeighborhood < 2) {
+            throw new IllegalArgumentException("clustersPerNeighborhood must be at least 2, got [" + clustersPerNeighborhood + "]");
+        }
+        doCluster(vectors, kMeansIntermediate, clustersPerNeighborhood, soarLambda);
+    }
+
+    private void doCluster(FloatVectorValues vectors, KMeansIntermediate kMeansIntermediate, int clustersPerNeighborhood, float soarLambda)
+        throws IOException {
         float[][] centroids = kMeansIntermediate.centroids();
+        boolean neighborAware = clustersPerNeighborhood != -1 && centroids.length > 1;
 
         List<NeighborHood> neighborhoods = null;
         // if there are very few centroids, don't bother with neighborhoods or neighbor aware clustering
@@ -308,11 +316,11 @@ void cluster(FloatVectorValues vectors, KMeansIntermediate kMeansIntermediate, b
             computeNeighborhoods(centroids, neighborhoods, clustersPerNeighborhood);
         }
         cluster(vectors, kMeansIntermediate, neighborhoods);
-        if (neighborAware && clustersPerNeighborhood > 0) {
+        if (neighborAware) {
             int[] assignments = kMeansIntermediate.assignments();
             assert assignments != null;
             assert assignments.length == vectors.size();
-            kMeansIntermediate.setSoarAssignments(assignSpilled(vectors, neighborhoods, centroids, assignments));
+            kMeansIntermediate.setSoarAssignments(assignSpilled(vectors, neighborhoods, centroids, assignments, soarLambda));
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

@@ -371,7 +371,7 @@ public Query regexpQuery(
     }
 
     public Query existsQuery(SearchExecutionContext context) {
-        if (hasDocValues() || getTextSearchInfo().hasNorms()) {
+        if (hasDocValues() || (isIndexed() && getTextSearchInfo().hasNorms())) {
             return new FieldExistsQuery(name());
         } else {
             return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name()));

server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java

@@ -52,6 +52,7 @@ private SearchCapabilities() {}
     private static final String SIGNIFICANT_TERMS_ON_NESTED_FIELDS = "significant_terms_on_nested_fields";
     private static final String EXCLUDE_VECTORS_PARAM = "exclude_vectors_param";
     private static final String DENSE_VECTOR_UPDATABLE_BBQ = "dense_vector_updatable_bbq";
+    private static final String FIELD_EXISTS_QUERY_FOR_TEXT_FIELDS_NO_INDEX_OR_DV = "field_exists_query_for_text_fields_no_index_or_dv";
 
     public static final Set<String> CAPABILITIES;
     static {
@@ -75,6 +76,7 @@ private SearchCapabilities() {}
         capabilities.add(SIGNIFICANT_TERMS_ON_NESTED_FIELDS);
         capabilities.add(EXCLUDE_VECTORS_PARAM);
         capabilities.add(DENSE_VECTOR_UPDATABLE_BBQ);
+        capabilities.add(FIELD_EXISTS_QUERY_FOR_TEXT_FIELDS_NO_INDEX_OR_DV);
         CAPABILITIES = Set.copyOf(capabilities);
     }
 }