Merge branch 'main' into simplify_esql_response

Author: idegtiarenko

docs/changelog/129418.yaml

@@ -0,0 +1,5 @@
+pr: 129418
+summary: Update traces duration mappings with appropriate unit type
+area: Ingest Node
+type: enhancement
+issues: []

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java

@@ -204,19 +204,58 @@ static float ipFloatBitImpl(float[] q, byte[] d, int start) {
         return acc0 + acc1 + acc2 + acc3;
     }
 
+    /**
+     * Returns the inner product (aka dot product) between the query vector {@code q}, and the data vector {@code d}.
+     * <p>
+     * The query vector should be {@link #B_QUERY}-bit quantized and striped, so that the first {@code n} bits
+     * of the array are the initial bits of each of the {@code n} vector dimensions; the next {@code n}
+     * bits are the second bits of each of the {@code n} vector dimensions, and so on
+     * (this algorithm is only valid for vectors with dimensions a multiple of 8).
+     * The striping is usually done by {@code BQSpaceUtils.transposeHalfByte}.
+     * <p>
+     * The data vector should be single-bit quantized.
+     *
+     * <h4>Dot products with bit quantization</h4>
+     *
+     * The dot product of any vector with a bit vector is a simple selector - each query vector dimension is multiplied
+     * by the 0 or 1 in the corresponding data vector dimension; the result is that each dimension value
+     * is either kept or ignored, with the dimensions that are kept then summed together.
+     *
+     * <h4>The algorithm</h4>
+     *
+     * The transposition already applied to the query vector ensures there's a 1-to-1 correspondence
+     * between the data vector bits and query vector bits (see {@code BQSpaceUtils.transposeHalfByte)};
+     * this means we can use a bitwise {@code &} to keep only the bits of the vector elements we want to sum.
+     * Essentially, the data vector is used as a selector for each of the striped bits of each vector dimension
+     * as stored, concatenated together, in {@code q}.
+     * <p>
+     * The final dot product result can be obtained by observing that the sum of each stripe of {@code n} bits
+     * can be computed using the bit count of that stripe. Similar to
+     * <a href="https://en.wikipedia.org/wiki/Multiplication_algorithm#Long_multiplication">long multiplication</a>,
+     * the result of each stripe of {@code n} bits can be added together by shifting the value {@code s} bits to the left,
+     * where {@code s} is the stripe number (0-3), then adding to the overall result. Any carry is handled by the add operation.
+     *
+     * @param q query vector, {@link #B_QUERY}-bit quantized and striped (see {@code BQSpaceUtils.transposeHalfByte})
+     * @param d data vector, 1-bit quantized
+     * @return  inner product result
+     */
     public static long ipByteBinByteImpl(byte[] q, byte[] d) {
         long ret = 0;
         int size = d.length;
-        for (int i = 0; i < B_QUERY; i++) {
+        for (int s = 0; s < B_QUERY; s++) { // for each stripe of B_QUERY-bit quantization in q...
             int r = 0;
-            long subRet = 0;
+            long stripeRet = 0;
+            // bitwise & the query and data vectors together, 32-bits at a time, and counting up the bits still set
             for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) {
-                subRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, i * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r));
+                stripeRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, s * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r));
             }
+            // handle any tail
+            // Java operations on bytes automatically extend to int, so we need to mask back down again in case it sign-extends the int
             for (; r < d.length; r++) {
-                subRet += Integer.bitCount((q[i * size + r] & d[r]) & 0xFF);
+                stripeRet += Integer.bitCount((q[s * size + r] & d[r]) & 0xFF);
             }
-            ret += subRet << i;
+            // shift the result of the s'th stripe s to the left and add to the result
+            ret += stripeRet << s;
         }
         return ret;
     }

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java

@@ -11,8 +11,16 @@
 
 public interface ESVectorUtilSupport {
 
+    /**
+     * The number of bits in bit-quantized query vectors
+     */
     short B_QUERY = 4;
 
+    /**
+     * Compute dot product between {@code q} and {@code d}
+     * @param q query vector, {@link #B_QUERY}-bit quantized and striped (see {@code BQSpaceUtils.transposeHalfByte})
+     * @param d data vector, 1-bit quantized
+     */
     long ipByteBinByte(byte[] q, byte[] d);
 
     int ipByteBit(byte[] q, byte[] d);

muted-tests.yml

@@ -544,6 +544,9 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
   method: testMergeTasksAreUnblockedWhenMoreDiskSpaceBecomesAvailable
   issue: https://github.com/elastic/elasticsearch/issues/129296
+- class: org.elasticsearch.xpack.security.PermissionsIT
+  method: testCanManageIndexWithNoPermissions
+  issue: https://github.com/elastic/elasticsearch/issues/129471
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/cluster/ProjectState.java

@@ -76,6 +76,13 @@ public ClusterState updatedState(Consumer<ProjectMetadata.Builder> projectBuilde
         return ClusterState.builder(cluster).putProjectMetadata(projectBuilder).build();
     }
 
+    /**
+     * Build a new {@link ClusterState} with the updated project.
+     */
+    public ClusterState updatedState(ProjectMetadata updatedProject) {
+        return ClusterState.builder(cluster).putProjectMetadata(updatedProject).build();
+    }
+
     /**
      * Build a new {@link ProjectState} with the updated {@code project}.
      */

x-pack/plugin/apm-data/src/main/resources/component-templates/[email protected]

@@ -43,6 +43,8 @@ template:
       # span.*
       span.duration.us:
         type: long
+        meta:
+          unit: micros
       span.representative_count:
         type: scaled_float
         scaling_factor: 1000
@@ -57,6 +59,8 @@ template:
         dynamic: false
       transaction.duration.us:
         type: long
+        meta:
+          unit: micros
       transaction.representative_count:
         type: scaled_float
         scaling_factor: 1000

x-pack/plugin/apm-data/src/main/resources/resources.yaml

@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin apm-data. This must be increased whenever an existing template or
 # pipeline is changed, in order for it to be updated on Elasticsearch upgrade.
-version: 14
+version: 15
 
 component-templates:
   # Data lifecycle.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java

@@ -223,6 +223,8 @@ static RoleDescriptor kibanaSystem(String name) {
                     .privileges("all")
                     .allowRestrictedIndices(true)
                     .build(),
+                // Fleet writes to this datastream for Agent status alerting feature
+                RoleDescriptor.IndicesPrivileges.builder().indices("logs-elastic_agent.status_change-*").privileges("all").build(),
                 // Fleet telemetry queries Agent Logs indices in kibana task runner
                 RoleDescriptor.IndicesPrivileges.builder().indices("logs-elastic_agent*").privileges("read").build(),
                 // Fleet publishes Agent metrics in kibana task runner

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/EsqlQueryGenerator.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.DropGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.EnrichGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.EvalGenerator;
+import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.ForkGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.GrokGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.KeepGenerator;
 import org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe.LimitGenerator;
@@ -53,6 +54,7 @@ public record QueryExecuted(String query, int depth, List<Column> outputSchema,
         DropGenerator.INSTANCE,
         EnrichGenerator.INSTANCE,
         EvalGenerator.INSTANCE,
+        ForkGenerator.INSTANCE,
         GrokGenerator.INSTANCE,
         KeepGenerator.INSTANCE,
         LimitGenerator.INSTANCE,

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/command/pipe/ForkGenerator.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.qa.rest.generative.command.pipe;
+
+import org.elasticsearch.xpack.esql.qa.rest.generative.EsqlQueryGenerator;
+import org.elasticsearch.xpack.esql.qa.rest.generative.command.CommandGenerator;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.ESTestCase.randomIntBetween;
+
+public class ForkGenerator implements CommandGenerator {
+
+    public static final String FORK = "fork";
+    public static final CommandGenerator INSTANCE = new ForkGenerator();
+
+    @Override
+    public CommandDescription generate(
+        List<CommandDescription> previousCommands,
+        List<EsqlQueryGenerator.Column> previousOutput,
+        QuerySchema schema
+    ) {
+        // FORK can only be allowed once - so we skip adding another FORK if we already have one
+        // otherwise, most generated queries would only result in a validation error
+        for (CommandDescription command : previousCommands) {
+            if (command.commandName().equals(FORK)) {
+                return new CommandDescription(FORK, this, " ", Map.of());
+            }
+        }
+
+        int n = randomIntBetween(2, 10);
+
+        String cmd = " | FORK " + "( WHERE true ) ".repeat(n) + " | WHERE _fork == \"fork" + randomIntBetween(1, n) + "\" | DROP _fork";
+
+        return new CommandDescription(FORK, this, cmd, Map.of());
+    }
+
+    @Override
+    public ValidationResult validateOutput(
+        List<CommandDescription> previousCommands,
+        CommandDescription command,
+        List<EsqlQueryGenerator.Column> previousColumns,
+        List<List<Object>> previousOutput,
+        List<EsqlQueryGenerator.Column> columns,
+        List<List<Object>> output
+    ) {
+        return CommandGenerator.expectSameRowCount(previousCommands, previousOutput, output);
+    }
+}