Merge branch 'main' into benchmark/main-enabled-skippers

Author: romseygeek

.buildkite/pipelines/periodic-fwc.template.yml

@@ -1,5 +1,5 @@
 steps:
-  - label: "{{matrix.FWC_VERSION}}" / fwc
+  - label: "{{matrix.FWC_VERSION}} / fwc"
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:

.buildkite/pipelines/periodic-fwc.yml

@@ -1,6 +1,6 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
 steps:
-  - label: "{{matrix.FWC_VERSION}}" / fwc
+  - label: "{{matrix.FWC_VERSION}} / fwc"
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 1, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 3, time = 1)
+// engage some noise reduction
+@Fork(value = 1)
+public class ComputeNeighboursBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    @Param({ "1000", "2000", "3000", "5000", "10000", "20000", "50000" })
+    int numVectors;
+
+    @Param({ "384", "782", "1024" })
+    int dims;
+
+    float[][] vectors;
+    int clusterPerNeighbour = 128;
+
+    @Setup
+    public void setup() throws IOException {
+        Random random = new Random(123);
+        vectors = new float[numVectors][dims];
+        for (float[] vector : vectors) {
+            for (int i = 0; i < dims; i++) {
+                vector[i] = random.nextFloat();
+            }
+        }
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void bruteForce(Blackhole bh) {
+        bh.consume(NeighborHood.computeNeighborhoodsBruteForce(vectors, clusterPerNeighbour));
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void graph(Blackhole bh) throws IOException {
+        bh.consume(NeighborHood.computeNeighborhoodsGraph(vectors, clusterPerNeighbour));
+    }
+}

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy

@@ -108,4 +108,65 @@ class ElasticsearchTestBasePluginFuncTest extends AbstractGradleFuncTest {
         then:
         result.task(':test').outcome == TaskOutcome.UP_TO_DATE
     }
+
+    def "uses new test seed for every invocation"() {
+        given:
+        file("src/test/java/acme/SomeTests.java").text = """
+
+        public class SomeTests {
+            @org.junit.Test
+            public void printTestSeed() {
+                System.out.println("TESTSEED=[" + System.getProperty("tests.seed") + "]");
+            }
+        }
+
+        """
+        buildFile.text = """
+            plugins {
+             id 'java'
+             id 'elasticsearch.test-base'
+            }
+
+            tasks.named('test').configure {
+                testLogging {
+                    showStandardStreams = true
+                }
+            }
+
+            tasks.register('test2', Test) {
+                classpath = sourceSets.test.runtimeClasspath
+                testClassesDirs = sourceSets.test.output.classesDirs
+                testLogging {
+                    showStandardStreams = true
+                }
+            }
+
+            repositories {
+                mavenCentral()
+            }
+
+            dependencies {
+                testImplementation 'junit:junit:4.12'
+            }
+
+        """
+
+        when:
+        def result1 = gradleRunner("cleanTest", "cleanTest2", "test", "test2").build()
+        def result2 = gradleRunner("cleanTest", "cleanTest2", "test", "test2").build()
+
+        then:
+        def seeds1 = result1.output.findAll(/(?m)TESTSEED=\[([^\]]+)\]/) { it[1] }
+        def seeds2 = result2.output.findAll(/(?m)TESTSEED=\[([^\]]+)\]/) { it[1] }
+
+        seeds1.unique().size() == 1
+        seeds2.unique().size() == 1
+
+        verifyAll {
+            seeds1[0] != null
+            seeds2[0] != null
+            seeds1[0] != seeds2[0]
+        }
+        result2.output.contains("Configuration cache entry reused.")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -24,6 +24,7 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.configuration.BuildFeatures;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.ProviderFactory;
@@ -56,6 +57,9 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
     @Inject
     protected abstract ProviderFactory getProviderFactory();
 
+    @Inject
+    protected abstract BuildFeatures getBuildFeatures();
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
@@ -164,9 +168,11 @@ public void execute(Task t) {
             );
             test.systemProperties(sysprops);
 
-            // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation
-            if (System.getProperty("ignore.tests.seed") != null) {
-                nonInputProperties.systemProperty("tests.seed", buildParams.get().getTestSeed());
+            // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability
+            // also ignore when configuration cache is on since the test seed as task input would break
+            // configuration cache reuse.
+            if (System.getProperty("ignore.tests.seed") != null || getBuildFeatures().getConfigurationCache().getActive().get()) {
+                nonInputProperties.systemProperty("tests.seed", buildParams.get().getTestSeedProvider());
             } else {
                 test.systemProperty("tests.seed", buildParams.get().getTestSeed());
             }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -56,6 +56,8 @@ public interface BuildParameterExtension {
 
     String getTestSeed();
 
+    Provider<String> getTestSeedProvider();
+
     Boolean getCi();
 
     Integer getDefaultParallel();
@@ -66,7 +68,7 @@ public interface BuildParameterExtension {
 
     Provider<BwcVersions> getBwcVersionsProvider();
 
-    Random getRandom();
+    Provider<Random> getRandom();
 
     Boolean getGraalVmRuntime();
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java

@@ -39,7 +39,7 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx
     private final Provider<String> gitRevision;
 
     private transient AtomicReference<ZonedDateTime> buildDate = new AtomicReference<>();
-    private final String testSeed;
+    private final Provider<String> testSeed;
     private final Boolean isCi;
     private final Integer defaultParallel;
     private final Boolean snapshotBuild;
@@ -58,7 +58,7 @@ public DefaultBuildParameterExtension(
         JavaVersion gradleJavaVersion,
         Provider<String> gitRevision,
         Provider<String> gitOrigin,
-        String testSeed,
+        Provider<String> testSeed,
         boolean isCi,
         int defaultParallel,
         final boolean isSnapshotBuild,
@@ -181,6 +181,11 @@ public ZonedDateTime getBuildDate() {
 
     @Override
     public String getTestSeed() {
+        return testSeed.get();
+    }
+
+    @Override
+    public Provider<String> getTestSeedProvider() {
         return testSeed;
     }
 
@@ -205,8 +210,8 @@ public BwcVersions getBwcVersions() {
     }
 
     @Override
-    public Random getRandom() {
-        return new Random(Long.parseUnsignedLong(testSeed.split(":")[0], 16));
+    public Provider<Random> getRandom() {
+        return getTestSeedProvider().map(seed -> new Random(Long.parseUnsignedLong(seed.split(":")[0], 16)));
     }
 
     @Override

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -29,6 +29,7 @@
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.configuration.BuildFeatures;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.model.ObjectFactory;
@@ -60,7 +61,6 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Properties;
-import java.util.Random;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -87,7 +87,7 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private final JavaInstallationRegistry javaInstallationRegistry;
     private final JvmMetadataDetector metadataDetector;
     private final ProviderFactory providers;
-    private final BranchesFileParser branchesFileParser;
+    private final BuildFeatures buildFeatures;
     private JavaToolchainService toolChainService;
     private Project project;
 
@@ -96,13 +96,14 @@ public GlobalBuildInfoPlugin(
         ObjectFactory objectFactory,
         JavaInstallationRegistry javaInstallationRegistry,
         JvmMetadataDetector metadataDetector,
-        ProviderFactory providers
+        ProviderFactory providers,
+        BuildFeatures buildFeatures
     ) {
         this.objectFactory = objectFactory;
         this.javaInstallationRegistry = javaInstallationRegistry;
         this.metadataDetector = new ErrorTraceMetadataDetector(metadataDetector);
         this.providers = providers;
-        this.branchesFileParser = new BranchesFileParser(new ObjectMapper());
+        this.buildFeatures = buildFeatures;
     }
 
     @Override
@@ -113,6 +114,7 @@ public void apply(Project project) {
         this.project = project;
         project.getPlugins().apply(JvmToolchainsPlugin.class);
         project.getPlugins().apply(JdkDownloadPlugin.class);
+        project.getPlugins().apply(VersionPropertiesPlugin.class);
         Provider<GitInfo> gitInfo = project.getPlugins().apply(GitInfoPlugin.class).getGitInfo();
 
         toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
@@ -121,11 +123,9 @@ public void apply(Project project) {
             throw new GradleException("Gradle " + minimumGradleVersion.getVersion() + "+ is required");
         }
 
-        project.getPlugins().apply(VersionPropertiesPlugin.class);
         Properties versionProperties = (Properties) project.getExtensions().getByName(VERSIONS_EXT);
         JavaVersion minimumCompilerVersion = JavaVersion.toVersion(versionProperties.get("minimumCompilerJava"));
         JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(versionProperties.get("minimumRuntimeJava"));
-
         Version elasticsearchVersionProperty = Version.fromString(versionProperties.getProperty("elasticsearch"));
 
         RuntimeJava runtimeJavaHome = findRuntimeJavaHome();
@@ -233,6 +233,7 @@ private List<DevelopmentBranch> getDevelopmentBranches() {
             }
         }
 
+        var branchesFileParser = new BranchesFileParser(new ObjectMapper());
         return branchesFileParser.parse(branchesBytes);
     }
 
@@ -266,7 +267,11 @@ private void logGlobalBuildInfo(BuildParameterExtension buildParams) {
         if (javaToolchainHome != null) {
             LOGGER.quiet("  JAVA_TOOLCHAIN_HOME   : " + javaToolchainHome);
         }
-        LOGGER.quiet("  Random Testing Seed   : " + buildParams.getTestSeed());
+
+        if (buildFeatures.getConfigurationCache().getActive().get() == false) {
+            // if configuration cache is enabled, resolving the test seed early breaks configuration cache reuse
+            LOGGER.quiet("  Random Testing Seed   : " + buildParams.getTestSeed());
+        }
         LOGGER.quiet("  In FIPS 140 mode      : " + buildParams.getInFipsJvm());
         LOGGER.quiet("=======================================");
     }
@@ -321,16 +326,8 @@ private Stream<InstallationLocation> getAvailableJavaInstallationLocationSteam()
         );
     }
 
-    private static String getTestSeed() {
-        String testSeedProperty = System.getProperty("tests.seed");
-        final String testSeed;
-        if (testSeedProperty == null) {
-            long seed = new Random(System.currentTimeMillis()).nextLong();
-            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT);
-        } else {
-            testSeed = testSeedProperty;
-        }
-        return testSeed;
+    private Provider<String> getTestSeed() {
+        return project.getProviders().of(TestSeedValueSource.class, spec -> {});
     }
 
     private static void throwInvalidJavaHomeException(String description, File javaHome, int expectedVersion, int actualVersion) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/TestSeedValueSource.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.info;
+
+import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
+
+import java.util.Locale;
+import java.util.Random;
+
+public abstract class TestSeedValueSource implements ValueSource<String, ValueSourceParameters.None> {
+
+    @Override
+    public ValueSourceParameters.None getParameters() {
+        return null;
+    }
+
+    @Override
+    public String obtain() {
+        String testSeedProperty = System.getProperty("tests.seed");
+        final String testSeed;
+        if (testSeedProperty == null) {
+            long seed = new Random(System.currentTimeMillis()).nextLong();
+            testSeed = Long.toUnsignedString(seed, 16).toUpperCase(Locale.ROOT);
+        } else {
+            testSeed = testSeedProperty;
+        }
+        return testSeed;
+    }
+}

build-tools-internal/version.properties

@@ -14,7 +14,7 @@ log4j             = 2.19.0
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
 jna               = 5.12.1
-netty             = 4.1.118.Final
+netty             = 4.1.126.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 awsv2sdk            = 2.31.78