Merge branch 'main' into esql_copy_sign_docs_and_fix

Author: nik9000

docs/changelog/132138.yaml

@@ -0,0 +1,6 @@
+pr: 132138
+summary: Fix lookup index resolution when field-caps returns empty mapping
+area: ES|QL
+type: bug
+issues:
+ - 132105

docs/changelog/132362.yaml

@@ -0,0 +1,5 @@
+pr: 132362
+summary: Inference API disable partial search results
+area: Machine Learning
+type: bug
+issues: []

muted-tests.yml

@@ -506,15 +506,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.AutodetectMemoryLimitIT
   method: testTooManyByAndOverFields
   issue: https://github.com/elastic/elasticsearch/issues/132310
-- class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testTermsQuery
-  issue: https://github.com/elastic/elasticsearch/issues/132335
-- class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusReindexedIntoStoredSourceChallengeRestIT
-  method: testTermsQuery
-  issue: https://github.com/elastic/elasticsearch/issues/132336
-- class: org.elasticsearch.xpack.logsdb.qa.LogsDbVersusReindexedLogsDbChallengeRestIT
-  method: testTermsQuery
-  issue: https://github.com/elastic/elasticsearch/issues/132337
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
   method: testBadAsyncId
   issue: https://github.com/elastic/elasticsearch/issues/132353

server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java

@@ -130,7 +130,7 @@ private void handleIncomingPublishRequest(
         ActionListener<PublishWithJoinResponse> publishResponseListener
     ) throws IOException {
         assert ThreadPool.assertCurrentThreadPool(GENERIC);
-        final Compressor compressor = CompressorFactory.compressor(request.bytes());
+        final Compressor compressor = CompressorFactory.compressorForUnknownXContentType(request.bytes());
         StreamInput in = request.bytes().streamInput();
         try {
             if (compressor != null) {

server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java

@@ -135,7 +135,7 @@ public CompressedXContent(ToXContent xcontent, ToXContent.Params params) throws
      * that may already be compressed.
      */
     public CompressedXContent(BytesReference data) throws IOException {
-        Compressor compressor = CompressorFactory.compressor(data);
+        Compressor compressor = CompressorFactory.compressorForUnknownXContentType(data);
         if (compressor != null) {
             // already compressed...
             this.bytes = BytesReference.toBytes(data);
@@ -148,7 +148,7 @@ public CompressedXContent(BytesReference data) throws IOException {
     }
 
     private void assertConsistent() {
-        assert CompressorFactory.compressor(new BytesArray(bytes)) != null;
+        assert CompressorFactory.compressorForUnknownXContentType(new BytesArray(bytes)) != null;
         assert this.sha256.equals(sha256(uncompressed()));
         assert this.sha256.equals(sha256FromCompressed(bytes));
     }

server/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java

@@ -22,7 +22,7 @@ public class CompressorFactory {
     public static final Compressor COMPRESSOR = new DeflateCompressor();
 
     public static boolean isCompressed(BytesReference bytes) {
-        return compressor(bytes) != null;
+        return compressorForUnknownXContentType(bytes) != null;
     }
 
     @Nullable
@@ -34,6 +34,15 @@ public static Compressor compressor(BytesReference bytes) {
             assert XContentHelper.xContentType(bytes) == null;
             return COMPRESSOR;
         }
+        return null;
+    }
+
+    @Nullable
+    public static Compressor compressorForUnknownXContentType(BytesReference bytes) {
+        Compressor compressor = compressor(bytes);
+        if (compressor != null) {
+            return compressor;
+        }
 
         XContentType contentType = XContentHelper.xContentType(bytes);
         if (contentType == null) {
@@ -56,13 +65,13 @@ private static boolean isAncient(BytesReference bytes) {
      * @throws NullPointerException a NullPointerException will be thrown when bytes is null
      */
     public static BytesReference uncompressIfNeeded(BytesReference bytes) throws IOException {
-        Compressor compressor = compressor(Objects.requireNonNull(bytes, "the BytesReference must not be null"));
+        Compressor compressor = compressorForUnknownXContentType(Objects.requireNonNull(bytes, "the BytesReference must not be null"));
         return compressor == null ? bytes : compressor.uncompress(bytes);
     }
 
     /** Decompress the provided {@link BytesReference}. */
     public static BytesReference uncompress(BytesReference bytes) throws IOException {
-        Compressor compressor = compressor(bytes);
+        Compressor compressor = compressorForUnknownXContentType(bytes);
         if (compressor == null) {
             throw new NotCompressedException();
         }

server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java

@@ -67,7 +67,7 @@ public static XContentParser createParser(NamedXContentRegistry registry, Deprec
      */
     @Deprecated
     public static XContentParser createParser(XContentParserConfiguration config, BytesReference bytes) throws IOException {
-        Compressor compressor = CompressorFactory.compressor(bytes);
+        Compressor compressor = CompressorFactory.compressorForUnknownXContentType(bytes);
         if (compressor != null) {
             InputStream compressedInput = compressor.threadLocalInputStream(bytes.streamInput());
             if (compressedInput.markSupported() == false) {
@@ -568,7 +568,7 @@ public interface CustomMerge {
     @Deprecated
     public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params)
         throws IOException {
-        Compressor compressor = CompressorFactory.compressor(source);
+        Compressor compressor = CompressorFactory.compressorForUnknownXContentType(source);
         if (compressor != null) {
             try (InputStream compressedStreamInput = compressor.threadLocalInputStream(source.streamInput())) {
                 builder.rawField(field, compressedStreamInput);
@@ -592,7 +592,7 @@ public static void writeRawField(
         ToXContent.Params params
     ) throws IOException {
         Objects.requireNonNull(xContentType);
-        Compressor compressor = CompressorFactory.compressor(source);
+        Compressor compressor = CompressorFactory.compressorForUnknownXContentType(source);
         if (compressor != null) {
             try (InputStream compressedStreamInput = compressor.threadLocalInputStream(source.streamInput())) {
                 builder.rawField(field, compressedStreamInput, xContentType);
@@ -677,7 +677,7 @@ public static BytesReference toXContent(ChunkedToXContent toXContent, XContentTy
      */
     @Deprecated
     public static XContentType xContentTypeMayCompressed(BytesReference bytes) {
-        Compressor compressor = CompressorFactory.compressor(bytes);
+        Compressor compressor = CompressorFactory.compressorForUnknownXContentType(bytes);
         if (compressor != null) {
             try {
                 InputStream compressedStreamInput = compressor.threadLocalInputStream(bytes.streamInput());

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/HierarchicalKMeans.java

@@ -13,6 +13,7 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Objects;
 
 /**
  * An implementation of the hierarchical k-means algorithm that better partitions data than naive k-means
@@ -148,30 +149,32 @@ static FloatVectorValues createClusterSlice(int clusterSize, int cluster, FloatV
     }
 
     void updateAssignmentsWithRecursiveSplit(KMeansIntermediate current, int cluster, KMeansIntermediate subPartitions) {
+        if (subPartitions.centroids().length == 0) {
+            return; // nothing to do, sub-partitions is empty
+        }
         int orgCentroidsSize = current.centroids().length;
         int newCentroidsSize = current.centroids().length + subPartitions.centroids().length - 1;
 
         // update based on the outcomes from the split clusters recursion
-        if (subPartitions.centroids().length > 1) {
-            float[][] newCentroids = new float[newCentroidsSize][dimension];
-            System.arraycopy(current.centroids(), 0, newCentroids, 0, current.centroids().length);
+        float[][] newCentroids = new float[newCentroidsSize][];
+        System.arraycopy(current.centroids(), 0, newCentroids, 0, current.centroids().length);
 
-            // replace the original cluster
-            int origCentroidOrd = 0;
-            newCentroids[cluster] = subPartitions.centroids()[0];
+        // replace the original cluster
+        int origCentroidOrd = 0;
+        newCentroids[cluster] = subPartitions.centroids()[0];
 
-            // append the remainder
-            System.arraycopy(subPartitions.centroids(), 1, newCentroids, current.centroids().length, subPartitions.centroids().length - 1);
+        // append the remainder
+        System.arraycopy(subPartitions.centroids(), 1, newCentroids, current.centroids().length, subPartitions.centroids().length - 1);
+        assert Arrays.stream(newCentroids).allMatch(Objects::nonNull);
 
-            current.setCentroids(newCentroids);
+        current.setCentroids(newCentroids);
 
-            for (int i = 0; i < subPartitions.assignments().length; i++) {
-                // this is a new centroid that was added, and so we'll need to remap it
-                if (subPartitions.assignments()[i] != origCentroidOrd) {
-                    int parentOrd = subPartitions.ordToDoc(i);
-                    assert current.assignments()[parentOrd] == cluster;
-                    current.assignments()[parentOrd] = subPartitions.assignments()[i] + orgCentroidsSize - 1;
-                }
+        for (int i = 0; i < subPartitions.assignments().length; i++) {
+            // this is a new centroid that was added, and so we'll need to remap it
+            if (subPartitions.assignments()[i] != origCentroidOrd) {
+                int parentOrd = subPartitions.ordToDoc(i);
+                assert current.assignments()[parentOrd] == cluster;
+                current.assignments()[parentOrd] = subPartitions.assignments()[i] + orgCentroidsSize - 1;
             }
         }
     }

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/KMeansLocal.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.index.codec.vectors.cluster;
 
 import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.hnsw.IntToIntFunction;
 import org.elasticsearch.index.codec.vectors.SampleReader;
@@ -70,17 +71,14 @@ private static boolean stepLloyd(
         FloatVectorValues vectors,
         IntToIntFunction translateOrd,
         float[][] centroids,
-        float[][] nextCentroids,
+        FixedBitSet centroidChanged,
+        int[] centroidCounts,
         int[] assignments,
         NeighborHood[] neighborhoods
     ) throws IOException {
         boolean changed = false;
         int dim = vectors.dimension();
-        int[] centroidCounts = new int[centroids.length];
-
-        for (float[] nextCentroid : nextCentroids) {
-            Arrays.fill(nextCentroid, 0.0f);
-        }
+        centroidChanged.clear();
         final float[] distances = new float[4];
         for (int idx = 0; idx < vectors.size(); idx++) {
             float[] vector = vectors.vectorValue(idx);
@@ -93,20 +91,39 @@ private static boolean stepLloyd(
                 bestCentroidOffset = getBestCentroid(centroids, vector, distances);
             }
             if (assignment != bestCentroidOffset) {
+                if (assignment != -1) {
+                    centroidChanged.set(assignment);
+                }
+                centroidChanged.set(bestCentroidOffset);
                 assignments[vectorOrd] = bestCentroidOffset;
                 changed = true;
             }
-            centroidCounts[bestCentroidOffset]++;
-            for (int d = 0; d < dim; d++) {
-                nextCentroids[bestCentroidOffset][d] += vector[d];
-            }
         }
+        if (changed) {
+            Arrays.fill(centroidCounts, 0);
+            for (int idx = 0; idx < vectors.size(); idx++) {
+                final int assignment = assignments[translateOrd.apply(idx)];
+                if (centroidChanged.get(assignment)) {
+                    float[] centroid = centroids[assignment];
+                    if (centroidCounts[assignment]++ == 0) {
+                        Arrays.fill(centroid, 0.0f);
+                    }
+                    float[] vector = vectors.vectorValue(idx);
+                    for (int d = 0; d < dim; d++) {
+                        centroid[d] += vector[d];
+                    }
+                }
+            }
 
-        for (int clusterIdx = 0; clusterIdx < centroids.length; clusterIdx++) {
-            if (centroidCounts[clusterIdx] > 0) {
-                float countF = (float) centroidCounts[clusterIdx];
-                for (int d = 0; d < dim; d++) {
-                    centroids[clusterIdx][d] = nextCentroids[clusterIdx][d] / countF;
+            for (int clusterIdx = 0; clusterIdx < centroids.length; clusterIdx++) {
+                if (centroidChanged.get(clusterIdx)) {
+                    float count = (float) centroidCounts[clusterIdx];
+                    if (count > 0) {
+                        float[] centroid = centroids[clusterIdx];
+                        for (int d = 0; d < dim; d++) {
+                            centroid[d] /= count;
+                        }
+                    }
                 }
             }
         }
@@ -420,17 +437,18 @@ private void cluster(FloatVectorValues vectors, KMeansIntermediate kMeansInterme
         }
 
         assert assignments.length == n;
-        float[][] nextCentroids = new float[centroids.length][vectors.dimension()];
+        FixedBitSet centroidChanged = new FixedBitSet(centroids.length);
+        int[] centroidCounts = new int[centroids.length];
         for (int i = 0; i < maxIterations; i++) {
             // This is potentially sampled, so we need to translate ordinals
-            if (stepLloyd(sampledVectors, translateOrd, centroids, nextCentroids, assignments, neighborhoods) == false) {
+            if (stepLloyd(sampledVectors, translateOrd, centroids, centroidChanged, centroidCounts, assignments, neighborhoods) == false) {
                 break;
             }
         }
         // If we were sampled, do a once over the full set of vectors to finalize the centroids
         if (sampleSize < n || maxIterations == 0) {
             // No ordinal translation needed here, we are using the full set of vectors
-            stepLloyd(vectors, i -> i, centroids, nextCentroids, assignments, neighborhoods);
+            stepLloyd(vectors, i -> i, centroids, centroidChanged, centroidCounts, assignments, neighborhoods);
         }
     }
 

server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTransportHandlerTests.java

@@ -145,7 +145,7 @@ private static boolean isDiff(BytesTransportRequest request, TransportVersion ve
             StreamInput in = null;
             try {
                 in = request.bytes().streamInput();
-                final Compressor compressor = CompressorFactory.compressor(request.bytes());
+                final Compressor compressor = CompressorFactory.compressorForUnknownXContentType(request.bytes());
                 if (compressor != null) {
                     in = compressor.threadLocalStreamInput(in);
                 }