Merge branch 'main' into jackson221

Author: jdconrad

.buildkite/pull-requests.json

@@ -47,7 +47,6 @@
       "set_commit_status": false,
       "build_on_commit": false,
       "build_on_comment": true,
-      "target_branch": "main",
       "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\S+)( please)?$"
     }
   ]

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 
 env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
 env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
-merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
+merge_base=$(git merge-base "origin/${GITHUB_PR_TARGET_BRANCH}" HEAD)
 
 # PR comment
 buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/ScalarOperations.java

@@ -17,6 +17,21 @@
  */
 class ScalarOperations {
 
+    static float cosine(byte[] a, byte[] b) {
+        int sum = 0;
+        int norm1 = 0;
+        int norm2 = 0;
+
+        for (int i = 0; i < a.length; i++) {
+            byte elem1 = a[i];
+            byte elem2 = b[i];
+            sum += elem1 * elem2;
+            norm1 += elem1 * elem1;
+            norm2 += elem2 * elem2;
+        }
+        return (float) (sum / Math.sqrt((double) norm1 * (double) norm2));
+    }
+
     static float dotProduct(float[] a, float[] b) {
         float res = 0;
         for (int i = 0; i < a.length; i++) {

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt8OperationBenchmark.java

@@ -62,7 +62,7 @@ public class VectorScorerInt8OperationBenchmark {
     @Param({ "1", "128", "207", "256", "300", "512", "702", "1024", "1536", "2048" })
     public int size;
 
-    @Param({ "DOT_PRODUCT", "EUCLIDEAN" })
+    @Param({ "COSINE", "DOT_PRODUCT", "EUCLIDEAN" })
     public VectorSimilarityType function;
 
     @FunctionalInterface
@@ -92,11 +92,13 @@ public void init() {
         MemorySegment.copy(MemorySegment.ofArray(bytesB), JAVA_BYTE, 0L, nativeSegB, JAVA_BYTE, 0L, bytesB.length);
 
         luceneImpl = switch (function) {
+            case COSINE -> VectorUtil::cosine;
             case DOT_PRODUCT -> VectorUtil::dotProduct;
             case EUCLIDEAN -> VectorUtil::squareDistance;
             default -> throw new UnsupportedOperationException("Not used");
         };
         nativeImpl = vectorSimilarityFunctions.getHandle(switch (function) {
+            case COSINE -> VectorSimilarityFunctions.Function.COSINE;
             case DOT_PRODUCT -> VectorSimilarityFunctions.Function.DOT_PRODUCT;
             case EUCLIDEAN -> VectorSimilarityFunctions.Function.SQUARE_DISTANCE;
             default -> throw new IllegalArgumentException(function.toString());
@@ -121,18 +123,18 @@ public float luceneWithCopy() {
     }
 
     @Benchmark
-    public int nativeWithNativeSeg() {
+    public float nativeWithNativeSeg() {
         try {
-            return (int) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
+            return (float) nativeImpl.invokeExact(nativeSegA, nativeSegB, size);
         } catch (Throwable t) {
             throw rethrow(t);
         }
     }
 
     @Benchmark
-    public int nativeWithHeapSeg() {
+    public float nativeWithHeapSeg() {
         try {
-            return (int) nativeImpl.invokeExact(heapSegA, heapSegB, size);
+            return (float) nativeImpl.invokeExact(heapSegA, heapSegB, size);
         } catch (Throwable t) {
             throw rethrow(t);
         }

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt8OperationBenchmarkTests.java

@@ -47,6 +47,7 @@ public void test() {
             bench.init();
             try {
                 float expected = switch (function) {
+                    case COSINE -> ScalarOperations.cosine(bench.bytesA, bench.bytesB);
                     case DOT_PRODUCT -> ScalarOperations.dotProduct(bench.bytesA, bench.bytesB);
                     case EUCLIDEAN -> ScalarOperations.squareDistance(bench.bytesA, bench.bytesB);
                     default -> throw new AssumptionViolatedException("Not tested");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -357,6 +357,23 @@ private static void configureEntitlements(Project project) {
                     "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
                         + modulesContainingEntitlementInstrumentation
                 );
+
+                // Export internal JDK packages that are required (temporarily) to declare instrumentation
+                test.jvmArgs("--add-exports=jdk.jlink/jdk.tools.jlink.internal=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=jdk.internal.vm.ci/jdk.vm.ci.services=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www.protocol.ftp=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www.protocol.file=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www.protocol.jar=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www.protocol.http=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www.protocol.https=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.net.www.protocol.mailto=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/sun.nio.ch=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/jdk.internal.foreign=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/jdk.internal.foreign.abi=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.base/jdk.internal.foreign.layout=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=java.net.http/jdk.internal.net.http=ALL-UNNAMED");
+                test.jvmArgs("--add-exports=jdk.jdi/com.sun.tools.jdi=ALL-UNNAMED");
             });
     }
 

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -43,4 +43,7 @@
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />
+
+  <!-- Entitlements rules API requires functional interfaces to be serializable -->
+  <suppress files="libs[/\\]entitlement[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]entitlement[/\\]rules[/\\]function[/\\]" checks="RegexpSinglelineJava" />
 </suppressions>

build-tools/build.gradle

@@ -34,6 +34,17 @@ java {
   sourceCompatibility = versions.get("minimumRuntimeJava")
 }
 
+sourceSets {
+  integTest {
+    compileClasspath += sourceSets["main"].output + configurations["testRuntimeClasspath"]
+    runtimeClasspath += output + compileClasspath
+  }
+}
+
+tasks.named("pluginUnderTestMetadata").configure {
+  getPluginClasspath().from(sourceSets.testFixtures.runtimeClasspath)
+}
+
 gradlePlugin {
     // We already configure publication and we don't need or want the one that comes
     // with the java-gradle-plugin
@@ -94,13 +105,6 @@ tasks.named("processResources").configure {
     }
 }
 
-sourceSets {
-    integTest {
-        compileClasspath += sourceSets["main"].output + configurations["testRuntimeClasspath"]
-        runtimeClasspath += output + compileClasspath
-    }
-}
-
 // we do not publish the test fixtures of build-tools
 components.java.withVariantsFromConfiguration(configurations.testFixturesApiElements) { skip() }
 components.java.withVariantsFromConfiguration(configurations.testFixturesRuntimeElements) { skip() }
@@ -140,6 +144,7 @@ dependencies {
     implementation buildLibs.jackson.core
     implementation buildLibs.jackson.databind
 
+    testFixturesApi buildLibs.commmons.io
     testFixturesApi gradleApi()
     testFixturesApi gradleTestKit()
     testFixturesApi buildLibs.junit
@@ -176,6 +181,7 @@ tasks.withType(JavaCompile).configureEach {
 tasks.named('test').configure {
     useJUnitPlatform()
 }
+
 tasks.register("integTest", Test) {
     testClassesDirs = sourceSets.integTest.output.classesDirs
     classpath = sourceSets.integTest.runtimeClasspath

build-tools/src/integTest/groovy/org/elasticsearch/gradle/JdkToolchainTestFixtureFuncTest.groovy

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import spock.lang.IgnoreIf
+
+import static org.elasticsearch.gradle.fixtures.JdkToolchainTestFixture.withMockedJdkDownload
+
+/**
+ * Integration tests for {@link org.elasticsearch.gradle.fixtures.JdkToolchainTestFixture}.
+ * Verifies that the fixture runs a Gradle build with a pre-installed fake JDK path and that
+ * the build succeeds. Full toolchain resolution and ES_JAVA_HOME override are covered by
+ * {@link TestClustersPluginFuncTest#override jdk usage via ES_JAVA_HOME for known jdk os incompatibilities}.
+ */
+@IgnoreIf({ os.isWindows() })
+class JdkToolchainTestFixtureFuncTest extends AbstractGradleFuncTest {
+
+    def "build with mocked JDK path succeeds"() {
+        when:
+        def runner = gradleRunner('tasks', '-i', '-Dorg.gradle.java.installations.auto-detect=false')
+        def result = withMockedJdkDownload(runner, 17, 'eclipse_adoptium')
+
+        then:
+        result.output.contains('BUILD SUCCESSFUL')
+    }
+}

build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -17,6 +17,7 @@ import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.GradleRunner
 
 import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.*
+import static org.elasticsearch.gradle.fixtures.JdkToolchainTestFixture.withMockedJdkDownload
 
 /**
  * We do not have coverage for the test cluster startup on windows yet.
@@ -241,13 +242,6 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
     @RestoreSystemProperties
     def "override jdk usage via ES_JAVA_HOME for known jdk os incompatibilities"() {
         given:
-
-        settingsFile.text = """
-            plugins {
-                id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0'
-            }
-            """ + settingsFile.text
-
         buildFile << """
             testClusters {
               myCluster {
@@ -266,16 +260,29 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
             }
         """
         when:
-        def result = withMockedDistributionDownload(
-            "8.10.4",
-            ElasticsearchDistribution.Platform.LINUX,
-            gradleRunner("myTask", '-Dos.name=Linux', '-Dos.version=6.14.0-1015-gcp', '-i')
-        ) {
-            build()
-        }
+        def runner = gradleRunner(
+            "myTask",
+            '-Dos.name=Linux',
+            '-Dos.version=6.14.0-1015-gcp',
+            '-i',
+            '-Dorg.gradle.java.installations.auto-detect=false'
+        )
+
+        def result = withMockedJdkDownload(runner, { GradleRunner r ->
+            withMockedDistributionDownload(
+                "8.10.4",
+                ElasticsearchDistribution.Platform.LINUX,
+                r,
+                { it.build() }
+            )
+        }, 17, "eclipse_adoptium")
 
         then:
-        result.output.lines().anyMatch { line -> line.startsWith("Running") && line.split().find { it.startsWith("ES_JAVA_HOME=") }.contains("eclipse_adoptium-17") }
+        def output = result.output
+        assert output.contains("Running")
+        assert output.lines().anyMatch { line ->
+            line.contains("ES_JAVA_HOME=") && line.contains("eclipse_adoptium-17")
+        }
     }
 
     boolean assertEsOutputContains(String testCluster, String expectedOutput) {