Merge branch 'main' into jackson221

Author: jdconrad

.gitignore

@@ -2,6 +2,9 @@
 # claude
 .claude
 
+# JetBrainsAI/Junie
+.junie/
+
 # intellij files
 .idea/
 *.iml

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -488,7 +488,7 @@ private void setupPages() {
                                     blockFactory.newConstantIntBlockWith(0, end - begin).asVector(),
                                     blockFactory.newConstantIntBlockWith(ctx.ord, end - begin).asVector(),
                                     docs.build(),
-                                    true
+                                    DocVector.config().singleSegmentNonDecreasing(true)
                                 ).asBlock()
                             )
                         );
@@ -525,7 +525,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                                         blockFactory.newConstantIntVector(0, size),
                                         leafs.build(),
                                         docs.build(),
-                                        null
+                                        DocVector.config()
                                     ).asBlock()
                                 )
                             );
@@ -543,7 +543,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                                 blockFactory.newConstantIntBlockWith(0, size).asVector(),
                                 leafs.build().asBlock().asVector(),
                                 docs.build(),
-                                null
+                                DocVector.config()
                             ).asBlock()
                         )
                     );
@@ -570,7 +570,7 @@ record ItrAndOrd(PrimitiveIterator.OfInt itr, int ord) {}
                                     blockFactory.newConstantIntVector(0, 1),
                                     blockFactory.newConstantIntVector(next.ord, 1),
                                     blockFactory.newConstantIntVector(next.itr.nextInt(), 1),
-                                    true
+                                    DocVector.config().singleSegmentNonDecreasing(true)
                                 ).asBlock()
                             )
                         );

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uOperationBenchmark.java

@@ -97,7 +97,7 @@ public void init() {
             case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
             case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
             default -> throw new IllegalArgumentException(function.toString());
-        }, VectorSimilarityFunctions.DataType.INT7, VectorSimilarityFunctions.Operation.SINGLE);
+        }, VectorSimilarityFunctions.DataType.INT7U, VectorSimilarityFunctions.Operation.SINGLE);
     }
 
     @TearDown

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt8OperationBenchmark.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.benchmark.vector.scorer;
+
+import org.apache.lucene.util.VectorUtil;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.logging.NodeNamePatternConverter;
+import org.elasticsearch.nativeaccess.NativeAccess;
+import org.elasticsearch.nativeaccess.VectorSimilarityFunctions;
+import org.elasticsearch.simdvec.VectorSimilarityType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.lang.invoke.MethodHandle;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.rethrow;
+
+@Fork(value = 3, jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+public class VectorScorerInt8OperationBenchmark {
+
+    static {
+        NodeNamePatternConverter.setGlobalNodeName("foo");
+        LogConfigurator.loadLog4jPlugins();
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    byte[] bytesA;
+    byte[] bytesB;
+    byte[] scratch;
+    MemorySegment heapSegA, heapSegB;
+    MemorySegment nativeSegA, nativeSegB;
+
+    Arena arena;
+
+    @Param({ "1", "128", "207", "256", "300", "512", "702", "1024", "1536", "2048" })
+    public int size;
+
+    @Param({ "DOT_PRODUCT", "EUCLIDEAN" })
+    public VectorSimilarityType function;
+
+    @FunctionalInterface
+    private interface LuceneFunction {
+        float run(byte[] vec1, byte[] vec2);
+    }
+
+    private LuceneFunction luceneImpl;
+    private MethodHandle nativeImpl;
+
+    @Setup(Level.Iteration)
+    public void init() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+
+        bytesA = new byte[size];
+        bytesB = new byte[size];
+        scratch = new byte[size];
+        random.nextBytes(bytesA);
+        random.nextBytes(bytesB);
+        heapSegA = MemorySegment.ofArray(bytesA);
+        heapSegB = MemorySegment.ofArray(bytesB);
+
+        arena = Arena.ofConfined();
+        nativeSegA = arena.allocate(bytesA.length);
+        MemorySegment.copy(MemorySegment.ofArray(bytesA), JAVA_BYTE, 0L, nativeSegA, JAVA_BYTE, 0L, bytesA.length);
+        nativeSegB = arena.allocate(bytesB.length);
+        MemorySegment.copy(MemorySegment.ofArray(bytesB), JAVA_BYTE, 0L, nativeSegB, JAVA_BYTE, 0L, bytesB.length);
+
+        luceneImpl = switch (function) {
+            case DOT_PRODUCT -> VectorUtil::dotProduct;
+            case EUCLIDEAN -> VectorUtil::squareDistance;
+            default -> throw new UnsupportedOperationException("Not used");
+        };
+        nativeImpl = vectorSimilarityFunctions.getHandle(switch (function) {
+            case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
+            case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
+            default -> throw new IllegalArgumentException(function.toString());
+        }, VectorSimilarityFunctions.DataType.INT8, VectorSimilarityFunctions.Operation.SINGLE);
+    }
+
+    @TearDown
+    public void teardown() {
+        arena.close();
+    }
+
+    @Benchmark
+    public float lucene() {
+        return luceneImpl.run(bytesA, bytesB);
+    }
+
+    @Benchmark
+    public float luceneWithCopy() {
+        // add a copy to better reflect what Lucene has to do to get the target vector on-heap
+        MemorySegment.copy(nativeSegB, JAVA_BYTE, 0L, scratch, 0, scratch.length);
+        return luceneImpl.run(bytesA, scratch);
+    }
+
+    @Benchmark
+    public int nativeWithNativeSeg() {
+        try {
+            return (int) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
+        }
+    }
+
+    @Benchmark
+    public int nativeWithHeapSeg() {
+        try {
+            return (int) nativeImpl.invokeExact(heapSegA, heapSegB, size);
+        } catch (Throwable t) {
+            throw rethrow(t);
+        }
+    }
+
+    static final VectorSimilarityFunctions vectorSimilarityFunctions = NativeAccess.instance().getVectorSimilarityFunctions().orElseThrow();
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerOSQBenchmark.java

@@ -198,7 +198,7 @@ public float[] score() throws IOException {
                     result.upperInterval(),
                     result.quantizedComponentSum(),
                     result.additionalCorrection(),
-                    VectorSimilarityFunction.EUCLIDEAN,
+                    similarityFunction,
                     centroidDp,
                     corrections[0],
                     corrections[1],

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt8OperationBenchmarkTests.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector.scorer;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.simdvec.VectorSimilarityType;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AssumptionViolatedException;
+import org.junit.BeforeClass;
+import org.openjdk.jmh.annotations.Param;
+
+import java.util.Arrays;
+
+import static org.elasticsearch.benchmark.vector.scorer.BenchmarkUtils.supportsHeapSegments;
+
+public class VectorScorerInt8OperationBenchmarkTests extends ESTestCase {
+
+    private final VectorSimilarityType function;
+    private final double delta;
+    private final int size;
+
+    public VectorScorerInt8OperationBenchmarkTests(VectorSimilarityType function, int size) {
+        this.function = function;
+        this.size = size;
+        delta = 1e-3 * size;
+    }
+
+    @BeforeClass
+    public static void skipWindows() {
+        assumeFalse("doesn't work on windows yet", Constants.WINDOWS);
+    }
+
+    public void test() {
+        for (int i = 0; i < 100; i++) {
+            var bench = new VectorScorerInt8OperationBenchmark();
+            bench.function = function;
+            bench.size = size;
+            bench.init();
+            try {
+                float expected = switch (function) {
+                    case DOT_PRODUCT -> ScalarOperations.dotProduct(bench.bytesA, bench.bytesB);
+                    case EUCLIDEAN -> ScalarOperations.squareDistance(bench.bytesA, bench.bytesB);
+                    default -> throw new AssumptionViolatedException("Not tested");
+                };
+                assertEquals(expected, bench.lucene(), delta);
+                assertEquals(expected, bench.luceneWithCopy(), delta);
+                assertEquals(expected, bench.nativeWithNativeSeg(), delta);
+                if (supportsHeapSegments()) {
+                    assertEquals(expected, bench.nativeWithHeapSeg(), delta);
+                }
+            } finally {
+                bench.teardown();
+            }
+        }
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parametersFactory() {
+        try {
+            String[] size = VectorScorerInt8OperationBenchmark.class.getField("size").getAnnotationsByType(Param.class)[0].value();
+            String[] functions = VectorScorerInt8OperationBenchmark.class.getField("function").getAnnotationsByType(Param.class)[0].value();
+            return () -> Arrays.stream(size)
+                .map(Integer::parseInt)
+                .flatMap(i -> Arrays.stream(functions).map(VectorSimilarityType::valueOf).map(f -> new Object[] { f, i }))
+                .iterator();
+        } catch (NoSuchFieldException e) {
+            throw new AssertionError(e);
+        }
+    }
+}

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerOSQBenchmarkTests.java

@@ -73,7 +73,7 @@ public void testSingleScalarVsVectorized() throws Exception {
 
                 float[] result = vectorized.score();
 
-                assertArrayEqualsPercent("single scoring, scalar VS vectorized", expected, result, deltaPercent);
+                assertArrayEqualsPercent("single scoring, scalar VS vectorized", expected, result, deltaPercent, DEFAULT_DELTA);
             } finally {
                 scalar.teardown();
                 vectorized.teardown();
@@ -93,6 +93,7 @@ public void testBulkScalarVsVectorized() throws Exception {
                 scalar.dims = dims;
                 scalar.bits = bits;
                 scalar.directoryType = directoryType;
+                scalar.similarityFunction = similarityFunction;
                 scalar.setup(new Random(seed));
 
                 float[] expected = scalar.bulkScore();
@@ -101,11 +102,12 @@ public void testBulkScalarVsVectorized() throws Exception {
                 vectorized.dims = dims;
                 vectorized.bits = bits;
                 vectorized.directoryType = directoryType;
+                vectorized.similarityFunction = similarityFunction;
                 vectorized.setup(new Random(seed));
 
                 float[] result = vectorized.bulkScore();
 
-                assertArrayEqualsPercent("bulk scoring, scalar VS vectorized", expected, result, deltaPercent);
+                assertArrayEqualsPercent("bulk scoring, scalar VS vectorized", expected, result, deltaPercent, DEFAULT_DELTA);
             } finally {
                 scalar.teardown();
                 vectorized.teardown();

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavadocPluginFuncTest.groovy

@@ -57,7 +57,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
         def options = normalized(file('some-depending-lib/build/tmp/javadoc/javadoc.options').text)
         options.contains('-notimestamp')
         options.contains('-quiet')
-        options.contains("-linkoffline '$expectedLink' './some-lib/build/docs/javadoc/'")
+        options.contains("-linkoffline '$expectedLink' '../some-lib/build/docs/javadoc/'")
 
         where:
         version        | versionType | expectedLink
@@ -131,7 +131,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
 
         // normal dependencies handles as usual
         result.task(":some-lib:javadoc").outcome == TaskOutcome.SUCCESS
-        options.contains("-linkoffline 'https://artifacts.elastic.co/javadoc/org/acme/some-lib/1.0' './some-lib/build/docs/javadoc/'")
+        options.contains("-linkoffline 'https://artifacts.elastic.co/javadoc/org/acme/some-lib/1.0' '../some-lib/build/docs/javadoc/'")
         file('some-depending-lib/build/docs/javadoc/org/acme/Something.html').exists() == false
 
         // source of shadowed dependencies are inlined
@@ -179,7 +179,7 @@ class ElasticsearchJavadocPluginFuncTest extends AbstractGradleFuncTest {
         def options = normalized(file('some-depending-lib/build/tmp/javadoc/javadoc.options').text)
         options.contains('-notimestamp')
         options.contains('-quiet')
-        options.contains("-linkoffline 'https://artifacts.elastic.co/javadoc/org/acme/some-lib/1.0' './some-lib/build/docs/javadoc'") == false
+        options.contains("-linkoffline 'https://artifacts.elastic.co/javadoc/org/acme/some-lib/1.0' '../some-lib/build/docs/javadoc'") == false
     }
 
     def "ensures module dependency in javadoc of projects"() {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java

@@ -115,20 +115,21 @@ private void configureDependency(Project project, boolean shadowed, ProjectDepen
                 var options = (StandardJavadocDocletOptions) javadoc.getOptions();
                 options.linksOffline(
                     artifactHost(project) + "/javadoc/" + artifactPath,
-                    upstreamProject.getBuildDir().getPath() + "/docs/javadoc/"
+                    project.getProjectDir().toPath().relativize(upstreamProject.getBuildDir().toPath()) + "/docs/javadoc/"
                 );
                 /*
                  *some dependent javadoc tasks are explicitly skipped. We need to ignore those external links as
                  * javadoc would fail otherwise.
                  * Using Action here instead of lambda to keep gradle happy and don't trigger deprecation
                  */
+                File projectDir = project.getProjectDir();
                 javadoc.doFirst(new Action<Task>() {
                     @Override
                     public void execute(Task task) {
                         List<JavadocOfflineLink> existingJavadocOfflineLinks = ((StandardJavadocDocletOptions) javadoc.getOptions())
                             .getLinksOffline()
                             .stream()
-                            .filter(javadocOfflineLink -> new File(javadocOfflineLink.getPackagelistLoc()).exists())
+                            .filter(javadocOfflineLink -> new File(projectDir, javadocOfflineLink.getPackagelistLoc()).exists())
                             .toList();
                         ((StandardJavadocDocletOptions) javadoc.getOptions()).setLinksOffline(existingJavadocOfflineLinks);
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java

@@ -38,10 +38,6 @@ void impl(String implName, Project implProject) {
         String capitalName = capitalize(projectName);
 
         Configuration implConfig = project.getConfigurations().detachedConfiguration(project.getDependencies().create(implProject));
-        implConfig.attributes(attrs -> {
-            attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_TYPE);
-            attrs.attribute(EmbeddedProviderPlugin.IMPL_ATTR, true);
-        });
 
         String manifestTaskName = "generate" + capitalName + "ProviderManifest";
         Provider<Directory> generatedResourcesRoot = project.getLayout().getBuildDirectory().dir("generated-resources");
@@ -51,11 +47,17 @@ void impl(String implName, Project implProject) {
             t.getProviderImplClasspath().from(implConfig);
         });
         String implTaskName = "generate" + capitalName + "ProviderImpl";
+
+        Configuration extractedImplConfig = implConfig.copy();
+        extractedImplConfig.attributes(attrs -> {
+            attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_TYPE);
+            attrs.attribute(EmbeddedProviderPlugin.IMPL_ATTR, true);
+        });
         var generateProviderImpl = project.getTasks().register(implTaskName, Sync.class);
         generateProviderImpl.configure(t -> {
             t.into(generatedResourcesRoot.map(d -> d.dir(implTaskName)));
             t.into("IMPL-JARS/" + implName, spec -> {
-                spec.from(implConfig);
+                spec.from(extractedImplConfig);
                 spec.from(generateProviderManifest);
             });
         });