Merge branch 'main' into 2025/09/09/cluster-applier-thread-watchdog

Author: DaveCTurner

.buildkite/pipelines/java-ea-check-new-build.yml

@@ -3,5 +3,5 @@ steps:
     env:
       RECENT_TIME_WINDOW: "24" # time window to consider a build as new in hours
     agents:
-      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.15"
       memory: "4G"

.buildkite/pipelines/pull-request/pr-upgrade.yml

@@ -2,5 +2,5 @@ steps:
   - label: pr-upgrade
     command: ".buildkite/scripts/run-pr-upgrade-tests.sh"
     agents:
-      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.15"
       memory: "4G"

.buildkite/pull-requests.json

@@ -18,6 +18,7 @@
     },
     {
       "enabled": true,
+      "target_branch": "main",
       "pipeline_slug": "elasticsearch-pull-request-transport-versions",
       "allow_org_users": true,
       "allowed_repo_permissions": [

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -42,6 +42,7 @@ public class OptimizedScalarQuantizerBenchmark {
     int dims;
 
     float[] vector;
+    float[] scratch;
     float[] centroid;
     int[] destination;
 
@@ -57,6 +58,7 @@ public void init() {
         destination = new int[dims];
         vector = new float[dims];
         centroid = new float[dims];
+        scratch = new float[dims];
         for (int i = 0; i < dims; ++i) {
             vector[i] = random.nextFloat();
             centroid[i] = random.nextFloat();
@@ -65,14 +67,14 @@ public void init() {
 
     @Benchmark
     public int[] scalar() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
+        osq.scalarQuantize(vector, scratch, destination, bits, centroid);
         return destination;
     }
 
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
     public int[] vector() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
+        osq.scalarQuantize(vector, scratch, destination, bits, centroid);
         return destination;
     }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -93,17 +93,29 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         assert file("myserver/src/main/resources/transport/definitions/referable/${name}.csv").exists() == false
     }
 
+    void assertUnreferableDefinition(String name, String content) {
+        File definitionFile = file("myserver/src/main/resources/transport/definitions/unreferable/${name}.csv")
+        assert definitionFile.exists()
+        assert definitionFile.text.strip() == content
+    }
+
     void assertUpperBound(String name, String content) {
         assert file("myserver/src/main/resources/transport/upper_bounds/${name}.csv").text.strip() == content
     }
 
+    void assertNoChanges() {
+        String output = execute("git diff")
+        assert output.strip().isEmpty() : "Expected no local git changes, but found:${System.lineSeparator()}${output}"
+    }
+
     def setup() {
         configurationCacheCompatible = false
         internalBuild()
         settingsFile << """
             include ':myserver'
             include ':myplugin'
         """
+        versionPropertiesFile.text = versionPropertiesFile.text.replace("9.1.0", "9.2.0")
 
         file("myserver/build.gradle") << """
             apply plugin: 'java-library'
@@ -116,10 +128,12 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         """
         referableTransportVersion("existing_91", "8012000")
         referableTransportVersion("existing_92", "8123000,8012001")
-        unreferableTransportVersion("initial_9_0_0", "8000000")
+        unreferableTransportVersion("initial_9.0.0", "8000000")
+        unreferableTransportVersion("initial_8.19.7", "7123001")
         transportVersionUpperBound("9.2", "existing_92", "8123000")
         transportVersionUpperBound("9.1", "existing_92", "8012001")
-        transportVersionUpperBound("9.0", "initial_9_0_0", "8000000")
+        transportVersionUpperBound("9.0", "initial_9.0.0", "8000000")
+        transportVersionUpperBound("8.19", "initial_8.19.7", "7123001")
         // a mock version of TransportVersion, just here so we can compile Dummy.java et al
         javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """
             public static TransportVersion fromName(String name) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionFuncTest.groovy

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport
+
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.TaskOutcome
+
+class GenerateInitialTransportVersionFuncTest extends AbstractTransportVersionFuncTest {
+    def runGenerateAndValidateTask(String... additionalArgs) {
+        List<String> args = new ArrayList<>()
+        args.add(":myserver:validateTransportVersionResources")
+        args.add(":myserver:generateInitialTransportVersion")
+        args.addAll(additionalArgs);
+        return gradleRunner(args.toArray())
+    }
+
+    def runGenerateTask(String... additionalArgs) {
+        List<String> args = new ArrayList<>()
+        args.add(":myserver:generateInitialTransportVersion")
+        args.addAll(additionalArgs);
+        return gradleRunner(args.toArray())
+    }
+
+    void assertGenerateSuccess(BuildResult result) {
+        assert result.task(":myserver:generateInitialTransportVersion").outcome == TaskOutcome.SUCCESS
+    }
+
+    void assertGenerateFailure(BuildResult result, String expectedOutput) {
+        assert result.task(":myserver:generateInitialTransportVersion").outcome == TaskOutcome.FAILED
+        assertOutputContains(result.output, expectedOutput)
+    }
+
+    void assertValidateSuccess(BuildResult result) {
+        assert result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
+
+    void assertGenerateAndValidateSuccess(BuildResult result) {
+        assertGenerateSuccess(result)
+        assertValidateSuccess(result)
+    }
+
+    def "setup is valid"() {
+        when:
+        def result = runGenerateAndValidateTask("--release-version", "9.0.0").build()
+
+        then:
+        assertGenerateAndValidateSuccess(result)
+        // should have been idempotent, nothing actually changed
+        assertNoChanges();
+    }
+
+    def "new minor also creates next upper bound"() {
+        given:
+        // version properties will be updated by release automation before running initial version generation
+        versionPropertiesFile.text = versionPropertiesFile.text.replace("9.2.0", "9.3.0")
+
+        when:
+        System.out.println("Running generation initial task")
+        def result = runGenerateAndValidateTask("--release-version", "9.2.0").build()
+        System.out.println("Done running generation task")
+
+        then:
+        assertGenerateAndValidateSuccess(result)
+        assertUnreferableDefinition("initial_9.2.0", "8124000")
+        assertUpperBound("9.2", "initial_9.2.0,8124000")
+        assertUpperBound("9.3", "initial_9.2.0,8124000")
+    }
+
+    def "patch updates existing upper bound"() {
+        when:
+        def result = runGenerateAndValidateTask("--release-version", "9.1.2").build()
+
+        then:
+        assertGenerateAndValidateSuccess(result)
+        assertUnreferableDefinition("initial_9.1.2", "8012002")
+        assertUpperBound("9.1", "initial_9.1.2,8012002")
+    }
+
+    def "cannot create upper bound file for patch"() {
+        when:
+        def result = runGenerateTask("--release-version", "9.3.7").buildAndFail()
+
+        then:
+        assertGenerateFailure(result, "Missing upper bound 9.3 for release version 9.3.7")
+    }
+}

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -412,7 +412,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
 
         then:
-        assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [9.0, 9.1, 9.2]")
+        assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [8.19, 9.0, 9.1, 9.2]")
     }
 
     def "name can be found from committed definition"() {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.elasticsearch.gradle.Version;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.provider.Property;
+import org.gradle.api.services.ServiceReference;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.options.Option;
+
+import java.io.IOException;
+import java.util.List;
+
+public abstract class GenerateInitialTransportVersionTask extends DefaultTask {
+
+    @ServiceReference("transportVersionResources")
+    abstract Property<TransportVersionResourcesService> getResourceService();
+
+    @Input
+    @Option(option = "release-version", description = "The Elasticsearch release version this transport version will be associated with")
+    public abstract Property<String> getReleaseVersion();
+
+    @Input
+    abstract Property<Version> getCurrentVersion();
+
+    @TaskAction
+    public void run() throws IOException {
+        Version releaseVersion = Version.fromString(getReleaseVersion().get());
+        String upperBoundName = getUpperBoundName(releaseVersion);
+        TransportVersionResourcesService resources = getResourceService().get();
+        TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromUpstream(upperBoundName);
+        String initialDefinitionName = "initial_" + releaseVersion;
+        TransportVersionDefinition existingDefinition = resources.getUnreferableDefinitionFromUpstream(initialDefinitionName);
+
+        if (existingDefinition != null) {
+            // this initial version has already been created upstream
+            return;
+        }
+
+        if (upstreamUpperBound == null) {
+            throw new RuntimeException("Missing upper bound " + upperBoundName + " for release version " + releaseVersion);
+        }
+        // minors increment by 1000 to create a unique base, patches increment by 1 as other patches do
+        int increment = releaseVersion.getRevision() == 0 ? 1000 : 1;
+        var id = TransportVersionId.fromInt(upstreamUpperBound.definitionId().complete() + increment);
+        var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id));
+        resources.writeUnreferableDefinition(definition);
+        var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id);
+        resources.writeUpperBound(newUpperBound);
+
+        if (releaseVersion.getRevision() == 0) {
+            Version currentVersion = getCurrentVersion().get();
+            String currentUpperBoundName = getUpperBoundName(currentVersion);
+            var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id);
+            resources.writeUpperBound(currentUpperBound);
+        }
+    }
+
+    private String getUpperBoundName(Version version) {
+        return version.getMajor() + "." + version.getMinor();
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java

@@ -10,6 +10,8 @@
 package org.elasticsearch.gradle.internal.transport;
 
 import org.elasticsearch.gradle.internal.ProjectSubscribeServicePlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -23,6 +25,7 @@ public class TransportVersionReferencesPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(LifecycleBasePlugin.class);
+        project.getPluginManager().apply(PrecommitTaskPlugin.class);
 
         project.getPlugins()
             .apply(ProjectSubscribeServicePlugin.class)
@@ -50,6 +53,6 @@ public void apply(Project project) {
                 t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file");
                 t.getReferencesFile().set(collectTask.get().getOutputFile());
             });
-        project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask));
+        project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -10,8 +10,10 @@
 package org.elasticsearch.gradle.internal.transport;
 
 import org.elasticsearch.gradle.Version;
-import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.ProjectSubscribeServicePlugin;
+import org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
@@ -20,6 +22,7 @@
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.util.Map;
+import java.util.Properties;
 
 public class TransportVersionResourcesPlugin implements Plugin<Project> {
 
@@ -28,7 +31,13 @@ public class TransportVersionResourcesPlugin implements Plugin<Project> {
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(LifecycleBasePlugin.class);
+        project.getPluginManager().apply(VersionPropertiesPlugin.class);
+        project.getPluginManager().apply(PrecommitTaskPlugin.class);
         var psService = project.getPlugins().apply(ProjectSubscribeServicePlugin.class).getService();
+
+        Properties versions = (Properties) project.getExtensions().getByName(VersionPropertiesPlugin.VERSIONS_EXT);
+        Version currentVersion = Version.fromString(versions.getProperty("elasticsearch"));
+
         var resourceRoot = getResourceRoot(project);
 
         String taskGroup = "Transport Versions";
@@ -61,7 +70,7 @@ public void apply(Project project) {
                 t.getShouldValidateDensity().convention(true);
                 t.getShouldValidatePrimaryIdNotPatch().convention(true);
             });
-        project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask));
+        project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
 
         var generateManifestTask = project.getTasks()
             .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> {
@@ -79,11 +88,17 @@ public void apply(Project project) {
                 t.setDescription("(Re)generates a transport version definition file");
                 t.getReferencesFiles().setFrom(tvReferencesConfig);
                 t.getIncrement().convention(1000);
-                Version esVersion = VersionProperties.getElasticsearchVersion();
-                t.getCurrentUpperBoundName().convention(esVersion.getMajor() + "." + esVersion.getMinor());
+                t.getCurrentUpperBoundName().convention(currentVersion.getMajor() + "." + currentVersion.getMinor());
             });
-
         validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask));
+
+        var generateInitialTask = project.getTasks()
+            .register("generateInitialTransportVersion", GenerateInitialTransportVersionTask.class, t -> {
+                t.setGroup(taskGroup);
+                t.setDescription("(Re)generates an initial transport version for an Elasticsearch release version");
+                t.getCurrentVersion().set(currentVersion);
+            });
+        validateTask.configure(t -> t.mustRunAfter(generateInitialTask));
     }
 
     private static String getResourceRoot(Project project) {