Merge branch 'main' into svilen/127208

Author: svilen-mihaylov-elastic

docs/changelog/131817.yaml

@@ -0,0 +1,5 @@
+pr: 131817
+summary: Change equals and hashcode for `ConstantNullBlock`
+area: ES|QL
+type: bug
+issues: []

docs/changelog/131917.yaml

@@ -0,0 +1,6 @@
+pr: 131917
+summary: Fix NPE on empty to_lower/to_upper call
+area: ES|QL
+type: bug
+issues:
+ - 131913

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ES91Int4VectorsScorer.java

@@ -161,8 +161,7 @@ public float applyCorrections(
         float qcDist
     ) {
         float ax = lowerInterval;
-        // Here we assume `lx` is simply bit vectors, so the scaling isn't necessary
-        float lx = upperInterval - ax;
+        float lx = (upperInterval - ax) * FOUR_BIT_SCALE;
         float ay = queryLowerInterval;
         float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE;
         float y1 = queryComponentSum;

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentES91Int4VectorsScorer.java

@@ -354,7 +354,7 @@ private void applyCorrectionsBulk(
                 memorySegment,
                 offset + 4 * BULK_SIZE + i * Float.BYTES,
                 ByteOrder.LITTLE_ENDIAN
-            ).sub(ax);
+            ).sub(ax).mul(FOUR_BIT_SCALE);
             var targetComponentSums = ShortVector.fromMemorySegment(
                 SHORT_SPECIES,
                 memorySegment,

libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/ES91Int4VectorScorerTests.java

@@ -20,7 +20,9 @@
 import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
 import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 
-import static org.hamcrest.Matchers.lessThan;
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.greaterThan;
 
 public class ES91Int4VectorScorerTests extends BaseVectorizationTests {
 
@@ -130,31 +132,59 @@ public void testInt4ScoreBulk() throws Exception {
         // only even dimensions are supported
         final int dimensions = random().nextInt(1, 1000) * 2;
         final int numVectors = random().nextInt(1, 10) * ES91Int4VectorsScorer.BULK_SIZE;
-        final byte[] vector = new byte[ES91Int4VectorsScorer.BULK_SIZE * dimensions];
-        final byte[] corrections = new byte[ES91Int4VectorsScorer.BULK_SIZE * 14];
+        final float[][] vectors = new float[numVectors][dimensions];
+        final int[] quantizedScratch = new int[dimensions];
+        final byte[] quantizeVector = new byte[dimensions];
+        final float[] centroid = new float[dimensions];
+        VectorSimilarityFunction similarityFunction = randomFrom(VectorSimilarityFunction.values());
+        for (int i = 0; i < dimensions; i++) {
+            centroid[i] = random().nextFloat();
+        }
+        if (similarityFunction != VectorSimilarityFunction.EUCLIDEAN) {
+            VectorUtil.l2normalize(centroid);
+        }
+
+        OptimizedScalarQuantizer quantizer = new OptimizedScalarQuantizer(similarityFunction);
         try (Directory dir = new MMapDirectory(createTempDir())) {
             try (IndexOutput out = dir.createOutput("tests.bin", IOContext.DEFAULT)) {
+                OptimizedScalarQuantizer.QuantizationResult[] results =
+                    new OptimizedScalarQuantizer.QuantizationResult[ES91Int4VectorsScorer.BULK_SIZE];
                 for (int i = 0; i < numVectors; i += ES91Int4VectorsScorer.BULK_SIZE) {
-                    for (int j = 0; j < ES91Int4VectorsScorer.BULK_SIZE * dimensions; j++) {
-                        vector[j] = (byte) random().nextInt(16); // 4-bit quantization
+                    for (int j = 0; j < ES91Int4VectorsScorer.BULK_SIZE; j++) {
+                        for (int k = 0; k < dimensions; k++) {
+                            vectors[i + j][k] = random().nextFloat();
+                        }
+                        if (similarityFunction != VectorSimilarityFunction.EUCLIDEAN) {
+                            VectorUtil.l2normalize(vectors[i + j]);
+                        }
+                        results[j] = quantizer.scalarQuantize(vectors[i + j].clone(), quantizedScratch, (byte) 4, centroid);
+                        for (int k = 0; k < dimensions; k++) {
+                            quantizeVector[k] = (byte) quantizedScratch[k];
+                        }
+                        out.writeBytes(quantizeVector, 0, dimensions);
                     }
-                    out.writeBytes(vector, 0, vector.length);
-                    random().nextBytes(corrections);
-                    out.writeBytes(corrections, 0, corrections.length);
+                    writeCorrections(results, out);
                 }
             }
-            final byte[] query = new byte[dimensions];
+            final float[] query = new float[dimensions];
+            final byte[] quantizeQuery = new byte[dimensions];
             for (int j = 0; j < dimensions; j++) {
-                query[j] = (byte) random().nextInt(16); // 4-bit quantization
+                query[j] = random().nextFloat();
             }
-            OptimizedScalarQuantizer.QuantizationResult queryCorrections = new OptimizedScalarQuantizer.QuantizationResult(
-                random().nextFloat(),
-                random().nextFloat(),
-                random().nextFloat(),
-                Short.toUnsignedInt((short) random().nextInt())
+            if (similarityFunction != VectorSimilarityFunction.EUCLIDEAN) {
+                VectorUtil.l2normalize(query);
+            }
+            OptimizedScalarQuantizer.QuantizationResult queryCorrections = quantizer.scalarQuantize(
+                query.clone(),
+                quantizedScratch,
+                (byte) 4,
+                centroid
             );
-            float centroidDp = random().nextFloat();
-            VectorSimilarityFunction similarityFunction = randomFrom(VectorSimilarityFunction.values());
+            for (int j = 0; j < dimensions; j++) {
+                quantizeQuery[j] = (byte) quantizedScratch[j];
+            }
+            float centroidDp = VectorUtil.dotProduct(centroid, centroid);
+
             try (IndexInput in = dir.openInput("tests.bin", IOContext.DEFAULT)) {
                 // Work on a slice that has just the right number of bytes to make the test fail with an
                 // index-out-of-bounds in case the implementation reads more than the allowed number of
@@ -166,7 +196,7 @@ public void testInt4ScoreBulk() throws Exception {
                 float[] scoresPanama = new float[ES91Int4VectorsScorer.BULK_SIZE];
                 for (int i = 0; i < numVectors; i += ES91Int4VectorsScorer.BULK_SIZE) {
                     defaultScorer.scoreBulk(
-                        query,
+                        quantizeQuery,
                         queryCorrections.lowerInterval(),
                         queryCorrections.upperInterval(),
                         queryCorrections.quantizedComponentSum(),
@@ -176,7 +206,7 @@ public void testInt4ScoreBulk() throws Exception {
                         scoresDefault
                     );
                     panamaScorer.scoreBulk(
-                        query,
+                        quantizeQuery,
                         queryCorrections.lowerInterval(),
                         queryCorrections.upperInterval(),
                         queryCorrections.quantizedComponentSum(),
@@ -186,29 +216,34 @@ public void testInt4ScoreBulk() throws Exception {
                         scoresPanama
                     );
                     for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
-                        if (scoresDefault[j] == scoresPanama[j]) {
-                            continue;
-                        }
-                        if (scoresDefault[j] > (1000 * Byte.MAX_VALUE)) {
-                            float diff = Math.abs(scoresDefault[j] - scoresPanama[j]);
-                            assertThat(
-                                "defaultScores: " + scoresDefault[j] + " bulkScores: " + scoresPanama[j],
-                                diff / scoresDefault[j],
-                                lessThan(1e-5f)
-                            );
-                            assertThat(
-                                "defaultScores: " + scoresDefault[j] + " bulkScores: " + scoresPanama[j],
-                                diff / scoresPanama[j],
-                                lessThan(1e-5f)
-                            );
-                        } else {
-                            assertEquals(scoresDefault[j], scoresPanama[j], 1e-2f);
-                        }
+                        assertEquals(scoresDefault[j], scoresPanama[j], 1e-2f);
+                        float realSimilarity = similarityFunction.compare(vectors[i + j], query);
+                        float accuracy = realSimilarity > scoresDefault[j]
+                            ? scoresDefault[j] / realSimilarity
+                            : realSimilarity / scoresDefault[j];
+                        assertThat(accuracy, greaterThan(0.90f));
                     }
                     assertEquals(in.getFilePointer(), slice.getFilePointer());
                 }
                 assertEquals((long) (dimensions + 14) * numVectors, in.getFilePointer());
             }
         }
     }
+
+    private static void writeCorrections(OptimizedScalarQuantizer.QuantizationResult[] corrections, IndexOutput out) throws IOException {
+        for (OptimizedScalarQuantizer.QuantizationResult correction : corrections) {
+            out.writeInt(Float.floatToIntBits(correction.lowerInterval()));
+        }
+        for (OptimizedScalarQuantizer.QuantizationResult correction : corrections) {
+            out.writeInt(Float.floatToIntBits(correction.upperInterval()));
+        }
+        for (OptimizedScalarQuantizer.QuantizationResult correction : corrections) {
+            int targetComponentSum = correction.quantizedComponentSum();
+            assert targetComponentSum >= 0 && targetComponentSum <= 0xffff;
+            out.writeShort((short) targetComponentSum);
+        }
+        for (OptimizedScalarQuantizer.QuantizationResult correction : corrections) {
+            out.writeInt(Float.floatToIntBits(correction.additionalCorrection()));
+        }
+    }
 }

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -47,6 +47,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInProgressException;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -1394,7 +1395,7 @@ public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception {
             .get();
 
         RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
-        SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
+        Snapshot snapshot = createSnapshotResponse.getSnapshotInfo().snapshot();
         assertEquals(RestStatus.OK, status);
 
         assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
@@ -1423,7 +1424,7 @@ public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception {
             client,
             request,
             "Snapshot ["
-                + snapshotId
+                + snapshot
                 + "] contains data stream ["
                 + datastreamName
                 + "] but custer does not have a matching index "

modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java

@@ -43,6 +43,7 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
         .setDescription("Onechat system index")
         .setType(Type.EXTERNAL_UNMANAGED)
         .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
+        .setAllowsTemplates()
         .build();
 
     public static final SystemIndexDescriptor APM_AGENT_CONFIG_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()

muted-tests.yml

@@ -274,9 +274,6 @@ tests:
 - class: org.elasticsearch.cli.keystore.AddStringKeyStoreCommandTests
   method: testStdinWithMultipleValues
   issue: https://github.com/elastic/elasticsearch/issues/126882
-- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2EnrichUnavailableRemotesIT
-  method: testEsqlEnrichWithSkipUnavailable
-  issue: https://github.com/elastic/elasticsearch/issues/127368
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header}
   issue: https://github.com/elastic/elasticsearch/issues/127625

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.snapshots.RestoreService;
@@ -29,14 +30,16 @@
 public class TransportRestoreSnapshotAction extends TransportMasterNodeAction<RestoreSnapshotRequest, RestoreSnapshotResponse> {
     public static final ActionType<RestoreSnapshotResponse> TYPE = new ActionType<>("cluster:admin/snapshot/restore");
     private final RestoreService restoreService;
+    private final ProjectResolver projectResolver;
 
     @Inject
     public TransportRestoreSnapshotAction(
         TransportService transportService,
         ClusterService clusterService,
         ThreadPool threadPool,
         RestoreService restoreService,
-        ActionFilters actionFilters
+        ActionFilters actionFilters,
+        ProjectResolver projectResolver
     ) {
         super(
             TYPE.name(),
@@ -49,13 +52,15 @@ public TransportRestoreSnapshotAction(
             threadPool.executor(ThreadPool.Names.SNAPSHOT_META)
         );
         this.restoreService = restoreService;
+        this.projectResolver = projectResolver;
     }
 
     @Override
     protected ClusterBlockException checkBlock(RestoreSnapshotRequest request, ClusterState state) {
         // Restoring a snapshot might change the global state and create/change an index,
         // so we need to check for METADATA_WRITE and WRITE blocks
-        ClusterBlockException blockException = state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
+        ClusterBlockException blockException = state.blocks()
+            .globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_WRITE);
         if (blockException != null) {
             return blockException;
         }
@@ -70,17 +75,21 @@ protected void masterOperation(
         final ClusterState state,
         final ActionListener<RestoreSnapshotResponse> listener
     ) {
-        restoreService.restoreSnapshot(request, listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> {
-            if (restoreCompletionResponse.restoreInfo() == null && request.waitForCompletion()) {
-                RestoreClusterStateListener.createAndRegisterListener(
-                    clusterService,
-                    restoreCompletionResponse,
-                    delegatedListener,
-                    threadPool.getThreadContext()
-                );
-            } else {
-                delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.restoreInfo()));
-            }
-        }));
+        restoreService.restoreSnapshot(
+            projectResolver.getProjectId(),
+            request,
+            listener.delegateFailure((delegatedListener, restoreCompletionResponse) -> {
+                if (restoreCompletionResponse.restoreInfo() == null && request.waitForCompletion()) {
+                    RestoreClusterStateListener.createAndRegisterListener(
+                        clusterService,
+                        restoreCompletionResponse,
+                        delegatedListener,
+                        threadPool.getThreadContext()
+                    );
+                } else {
+                    delegatedListener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.restoreInfo()));
+                }
+            })
+        );
     }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/recovery/TransportRecoveryAction.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -43,14 +44,16 @@
 public class TransportRecoveryAction extends TransportBroadcastByNodeAction<RecoveryRequest, RecoveryResponse, RecoveryState> {
 
     private final IndicesService indicesService;
+    private final ProjectResolver projectResolver;
 
     @Inject
     public TransportRecoveryAction(
         ClusterService clusterService,
         TransportService transportService,
         IndicesService indicesService,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        ProjectResolver projectResolver
     ) {
         super(
             RecoveryAction.NAME,
@@ -62,6 +65,7 @@ public TransportRecoveryAction(
             transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT)
         );
         this.indicesService = indicesService;
+        this.projectResolver = projectResolver;
     }
 
     @Override
@@ -110,16 +114,16 @@ protected void shardOperation(RecoveryRequest request, ShardRouting shardRouting
 
     @Override
     protected ShardsIterator shards(ClusterState state, RecoveryRequest request, String[] concreteIndices) {
-        return state.routingTable().allShardsIncludingRelocationTargets(concreteIndices);
+        return state.routingTable(projectResolver.getProjectId()).allShardsIncludingRelocationTargets(concreteIndices);
     }
 
     @Override
     protected ClusterBlockException checkGlobalBlock(ClusterState state, RecoveryRequest request) {
-        return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ);
+        return state.blocks().globalBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ);
     }
 
     @Override
     protected ClusterBlockException checkRequestBlock(ClusterState state, RecoveryRequest request, String[] concreteIndices) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices);
+        return state.blocks().indicesBlockedException(projectResolver.getProjectId(), ClusterBlockLevel.METADATA_READ, concreteIndices);
     }
 }