Merge branch 'main' into pkar/msearch-flatworld-search-project-routing

Author: pawankartik-elastic

BUILDING.md

@@ -239,7 +239,7 @@ similar to how the Gradle build-in `java-test-fixtures` plugin works.
 ```
 dependencies {
   testImplementation(project(":fixture-providing-project')) {
-    requireCapabilities("org.elasticsearch.gradle:fixture-providing-project-test-artifacts")
+    requireCapabilities(${project(":fixture-providing-project').group}:fixture-providing-project-test-artifacts")
   }
 }
 ```

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uBulkBenchmark.java

@@ -14,6 +14,7 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -48,6 +49,7 @@
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.createRandomInt7VectorData;
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.getScorerFactoryOrDie;
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.luceneScoreSupplier;
+import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.luceneScorer;
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.readNodeCorrectionConstant;
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.supportsHeapSegments;
 import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.vectorValues;
@@ -80,10 +82,10 @@ public class VectorScorerInt7uBulkBenchmark {
 
     // 128k is typically enough to not fit in L1 (core) cache for most processors;
     // 1.5M is typically enough to not fit in L2 (core) cache;
-    // 40M is typically enough to not fit in L3 cache
-    @Param({ "128000", "1500000", "30000000" })
+    // 130M is enough to not fit in L3 cache
+    @Param({ "128", "1500", "130000" })
     public int numVectors;
-    public int numVectorsToScore = 20_000;
+    public int numVectorsToScore;
 
     Path path;
     Directory dir;
@@ -100,8 +102,12 @@ public class VectorScorerInt7uBulkBenchmark {
     UpdateableRandomVectorScorer luceneDotScorer;
     UpdateableRandomVectorScorer nativeDotScorer;
 
+    RandomVectorScorer luceneDotScorerQuery;
+    RandomVectorScorer nativeDotScorerQuery;
+
     @Setup(Level.Trial)
     public void setup() throws IOException {
+        numVectorsToScore = Math.min(numVectors, 20_000);
         factory = getScorerFactoryOrDie();
 
         var random = ThreadLocalRandom.current();
@@ -127,6 +133,17 @@ public void setup() throws IOException {
             .orElseThrow()
             .scorer();
         nativeDotScorer.setScoringOrdinal(targetOrd);
+
+        if (supportsHeapSegments()) {
+            // setup for getInt7SQVectorScorer / query vector scoring
+            float[] queryVec = new float[dims];
+            for (int i = 0; i < dims; i++) {
+                queryVec[i] = random.nextFloat();
+            }
+            luceneDotScorerQuery = luceneScorer(dotProductValues, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
+            nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, dotProductValues, queryVec)
+                .orElseThrow();
+        }
     }
 
     @TearDown
@@ -151,6 +168,14 @@ public float[] dotProductLuceneMultipleRandom() throws IOException {
         return scores;
     }
 
+    @Benchmark
+    public float[] dotProductLuceneQueryMultipleRandom() throws IOException {
+        for (int v = 0; v < numVectorsToScore; v++) {
+            scores[v] = luceneDotScorerQuery.score(ordinals[v]);
+        }
+        return scores;
+    }
+
     @Benchmark
     public float[] dotProductNativeMultipleSequential() throws IOException {
         for (int v = 0; v < numVectorsToScore; v++) {
@@ -167,6 +192,14 @@ public float[] dotProductNativeMultipleRandom() throws IOException {
         return scores;
     }
 
+    @Benchmark
+    public float[] dotProductNativeQueryMultipleRandom() throws IOException {
+        for (int v = 0; v < numVectorsToScore; v++) {
+            scores[v] = nativeDotScorerQuery.score(ordinals[v]);
+        }
+        return scores;
+    }
+
     @Benchmark
     public float[] dotProductNativeMultipleSequentialBulk() throws IOException {
         nativeDotScorer.bulkScore(ids, scores, ordinals.length);
@@ -179,6 +212,12 @@ public float[] dotProductNativeMultipleRandomBulk() throws IOException {
         return scores;
     }
 
+    @Benchmark
+    public float[] dotProductNativeQueryMultipleRandomBulk() throws IOException {
+        nativeDotScorerQuery.bulkScore(ordinals, scores, ordinals.length);
+        return scores;
+    }
+
     @Benchmark
     public float[] dotProductScalarMultipleSequential() throws IOException {
         var queryVector = dotProductValues.vectorValue(targetOrd);

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uBulkBenchmarkTests.java

@@ -18,6 +18,8 @@
 
 import java.util.Arrays;
 
+import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.supportsHeapSegments;
+
 public class VectorScorerInt7uBulkBenchmarkTests extends ESTestCase {
 
     final float delta = 1e-3f;
@@ -61,6 +63,11 @@ public void testDotProductRandom() throws Exception {
                 assertArrayEquals(expected, bench.dotProductLuceneMultipleRandom(), delta);
                 assertArrayEquals(expected, bench.dotProductNativeMultipleRandom(), delta);
                 assertArrayEquals(expected, bench.dotProductNativeMultipleRandomBulk(), delta);
+                if (supportsHeapSegments()) {
+                    assertArrayEquals(expected, bench.dotProductLuceneQueryMultipleRandom(), delta);
+                    assertArrayEquals(expected, bench.dotProductNativeQueryMultipleRandom(), delta);
+                    assertArrayEquals(expected, bench.dotProductNativeQueryMultipleRandomBulk(), delta);
+                }
             } finally {
                 bench.teardown();
             }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java

@@ -34,7 +34,7 @@ public void registerTestArtifactFromSourceSet(SourceSet sourceSet) {
         JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
         javaPluginExtension.registerFeature(name + "Artifacts", featureSpec -> {
             featureSpec.usingSourceSet(sourceSet);
-            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-" + sourceSet.getName() + "-artifacts", "1.0");
+            featureSpec.capability(project.getGroup().toString(), project.getName() + "-" + sourceSet.getName() + "-artifacts", "1.0");
             // This feature is only used internally in the
             // elasticsearch build so we do not need any publication.
             featureSpec.disablePublication();

build.gradle

@@ -95,7 +95,7 @@ tasks.named('zipAggregation').configure {
 ext.testArtifact = { p, String name = "test" ->
   def projectDependency = p.dependencies.create(p)
   projectDependency.capabilities {
-    requireCapabilities("org.elasticsearch.gradle:${projectDependency.name}-${name}-artifacts")
+    requireCapabilities("${p.group}:${projectDependency.name}-${name}-artifacts")
   };
 }
 

docs/changelog/137598.yaml

@@ -0,0 +1,6 @@
+pr: 137598
+summary: Improve SAML error handling by adding metadata
+area: Authentication
+type: enhancement
+issues:
+ - 128179

docs/changelog/138072.yaml

@@ -0,0 +1,5 @@
+pr: 138072
+summary: Add a new setting for s3 API call timeout
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/138198.yaml

@@ -0,0 +1,5 @@
+pr: 138198
+summary: Add Embedding inference task type
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/138260.yaml

@@ -0,0 +1,6 @@
+pr: 138260
+summary: "[HealthAPI] Deterministic shard availability key order"
+area: Health
+type: enhancement
+issues:
+ - 138043

docs/changelog/138332.yaml

@@ -0,0 +1,5 @@
+pr: 138332
+summary: Do not skip a remote cluster base on the query's execution time status
+area: ES|QL
+type: bug
+issues: []