Merge branch 'main' into mis-enable-failure-store

Author: rubvs

docs/changelog/127621.yaml

@@ -0,0 +1,5 @@
+pr: 127621
+summary: Fix error message when changing the password for a user in the file realm
+area: Security
+type: bug
+issues: []

docs/changelog/129161.yaml

@@ -0,0 +1,5 @@
+pr: 129161
+summary: Add Telemetry for models without adaptive allocations
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/129418.yaml

@@ -0,0 +1,5 @@
+pr: 129418
+summary: Update traces duration mappings with appropriate unit type
+area: Ingest Node
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/keyword.md

@@ -295,14 +295,14 @@ It is both allowed to submit documents that don’t have a value for the field o
 ```console
 POST logs-debug/_doc
 {
-  "date": "2019-12-12",
+  "@timestamp": "2019-12-12",
   "message": "Starting up Elasticsearch",
   "level": "debug"
 }
 
 POST logs-debug/_doc
 {
-  "date": "2019-12-12",
+  "@timestamp": "2019-12-12",
   "message": "Starting up Elasticsearch"
 }
 ```

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java

@@ -204,19 +204,58 @@ static float ipFloatBitImpl(float[] q, byte[] d, int start) {
         return acc0 + acc1 + acc2 + acc3;
     }
 
+    /**
+     * Returns the inner product (aka dot product) between the query vector {@code q}, and the data vector {@code d}.
+     * <p>
+     * The query vector should be {@link #B_QUERY}-bit quantized and striped, so that the first {@code n} bits
+     * of the array are the initial bits of each of the {@code n} vector dimensions; the next {@code n}
+     * bits are the second bits of each of the {@code n} vector dimensions, and so on
+     * (this algorithm is only valid for vectors with dimensions a multiple of 8).
+     * The striping is usually done by {@code BQSpaceUtils.transposeHalfByte}.
+     * <p>
+     * The data vector should be single-bit quantized.
+     *
+     * <h4>Dot products with bit quantization</h4>
+     *
+     * The dot product of any vector with a bit vector is a simple selector - each query vector dimension is multiplied
+     * by the 0 or 1 in the corresponding data vector dimension; the result is that each dimension value
+     * is either kept or ignored, with the dimensions that are kept then summed together.
+     *
+     * <h4>The algorithm</h4>
+     *
+     * The transposition already applied to the query vector ensures there's a 1-to-1 correspondence
+     * between the data vector bits and query vector bits (see {@code BQSpaceUtils.transposeHalfByte)};
+     * this means we can use a bitwise {@code &} to keep only the bits of the vector elements we want to sum.
+     * Essentially, the data vector is used as a selector for each of the striped bits of each vector dimension
+     * as stored, concatenated together, in {@code q}.
+     * <p>
+     * The final dot product result can be obtained by observing that the sum of each stripe of {@code n} bits
+     * can be computed using the bit count of that stripe. Similar to
+     * <a href="https://en.wikipedia.org/wiki/Multiplication_algorithm#Long_multiplication">long multiplication</a>,
+     * the result of each stripe of {@code n} bits can be added together by shifting the value {@code s} bits to the left,
+     * where {@code s} is the stripe number (0-3), then adding to the overall result. Any carry is handled by the add operation.
+     *
+     * @param q query vector, {@link #B_QUERY}-bit quantized and striped (see {@code BQSpaceUtils.transposeHalfByte})
+     * @param d data vector, 1-bit quantized
+     * @return  inner product result
+     */
     public static long ipByteBinByteImpl(byte[] q, byte[] d) {
         long ret = 0;
         int size = d.length;
-        for (int i = 0; i < B_QUERY; i++) {
+        for (int s = 0; s < B_QUERY; s++) { // for each stripe of B_QUERY-bit quantization in q...
             int r = 0;
-            long subRet = 0;
+            long stripeRet = 0;
+            // bitwise & the query and data vectors together, 32-bits at a time, and counting up the bits still set
             for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) {
-                subRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, i * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r));
+                stripeRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, s * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r));
             }
+            // handle any tail
+            // Java operations on bytes automatically extend to int, so we need to mask back down again in case it sign-extends the int
             for (; r < d.length; r++) {
-                subRet += Integer.bitCount((q[i * size + r] & d[r]) & 0xFF);
+                stripeRet += Integer.bitCount((q[s * size + r] & d[r]) & 0xFF);
             }
-            ret += subRet << i;
+            // shift the result of the s'th stripe s to the left and add to the result
+            ret += stripeRet << s;
         }
         return ret;
     }

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java

@@ -11,8 +11,16 @@
 
 public interface ESVectorUtilSupport {
 
+    /**
+     * The number of bits in bit-quantized query vectors
+     */
     short B_QUERY = 4;
 
+    /**
+     * Compute dot product between {@code q} and {@code d}
+     * @param q query vector, {@link #B_QUERY}-bit quantized and striped (see {@code BQSpaceUtils.transposeHalfByte})
+     * @param d data vector, 1-bit quantized
+     */
     long ipByteBinByte(byte[] q, byte[] d);
 
     int ipByteBit(byte[] q, byte[] d);

muted-tests.yml

@@ -529,6 +529,24 @@ tests:
 - class: org.elasticsearch.entitlement.runtime.policy.FileAccessTreeTests
   method: testWindowsAbsolutPathAccess
   issue: https://github.com/elastic/elasticsearch/issues/129168
+- class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
+  method: testCaseInsensitiveEquality {KEYWORD}
+  issue: https://github.com/elastic/elasticsearch/issues/129422
+- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
+  method: test {knn-function.KnnSearchWithKOption ASYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/129447
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/129453
+- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
+  method: testWithDatastreams
+  issue: https://github.com/elastic/elasticsearch/issues/129457
+- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
+  method: testMergeTasksAreUnblockedWhenMoreDiskSpaceBecomesAvailable
+  issue: https://github.com/elastic/elasticsearch/issues/129296
+- class: org.elasticsearch.xpack.security.PermissionsIT
+  method: testCanManageIndexWithNoPermissions
+  issue: https://github.com/elastic/elasticsearch/issues/129471
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/cluster/ProjectState.java

@@ -76,6 +76,13 @@ public ClusterState updatedState(Consumer<ProjectMetadata.Builder> projectBuilde
         return ClusterState.builder(cluster).putProjectMetadata(projectBuilder).build();
     }
 
+    /**
+     * Build a new {@link ClusterState} with the updated project.
+     */
+    public ClusterState updatedState(ProjectMetadata updatedProject) {
+        return ClusterState.builder(cluster).putProjectMetadata(updatedProject).build();
+    }
+
     /**
      * Build a new {@link ProjectState} with the updated {@code project}.
      */

server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java

@@ -58,6 +58,7 @@ public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard,
             // shrink/split/clone operation is going to clone existing locally placed shards using file system hard links
             // so no additional space is going to be used until future merges
             case LOCAL_SHARDS -> false;
+            case RESHARD_SPLIT -> false;
         };
     }
 

server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.xcontent.ToXContent;
@@ -31,6 +32,7 @@
  * - {@link PeerRecoverySource} recovery from a primary on another node
  * - {@link SnapshotRecoverySource} recovery from a snapshot
  * - {@link LocalShardsRecoverySource} recovery from other shards of another index on the same node
+ * - {@link ReshardSplitRecoverySource} recovery of a shard that is created as a result of a resharding split
  */
 public abstract class RecoverySource implements Writeable, ToXContentObject {
 
@@ -57,6 +59,7 @@ public static RecoverySource readFrom(StreamInput in) throws IOException {
             case PEER -> PeerRecoverySource.INSTANCE;
             case SNAPSHOT -> new SnapshotRecoverySource(in);
             case LOCAL_SHARDS -> LocalShardsRecoverySource.INSTANCE;
+            case RESHARD_SPLIT -> new ReshardSplitRecoverySource(in);
         };
     }
 
@@ -78,7 +81,8 @@ public enum Type {
         EXISTING_STORE,
         PEER,
         SNAPSHOT,
-        LOCAL_SHARDS
+        LOCAL_SHARDS,
+        RESHARD_SPLIT
     }
 
     public abstract Type getType();
@@ -319,4 +323,39 @@ public boolean expectEmptyRetentionLeases() {
             return false;
         }
     }
+
+    /**
+     * Recovery of a shard that is created as a result of a resharding split.
+     * Not to be confused with _split API.
+     */
+    public static class ReshardSplitRecoverySource extends RecoverySource {
+        private final ShardId sourceShardId;
+
+        public ReshardSplitRecoverySource(ShardId sourceShardId) {
+            this.sourceShardId = sourceShardId;
+        }
+
+        ReshardSplitRecoverySource(StreamInput in) throws IOException {
+            sourceShardId = new ShardId(in);
+        }
+
+        @Override
+        public Type getType() {
+            return Type.RESHARD_SPLIT;
+        }
+
+        public ShardId getSourceShardId() {
+            return sourceShardId;
+        }
+
+        @Override
+        protected void writeAdditionalFields(StreamOutput out) throws IOException {
+            sourceShardId.writeTo(out);
+        }
+
+        @Override
+        public void addAdditionalFields(XContentBuilder builder, Params params) throws IOException {
+            sourceShardId.toXContent(builder, params);
+        }
+    }
 }