Merge branch '9.0' into backport-9.0-esql-async-headers-fix

Author: ivancea

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/ResolveTransportVersionConflictFuncTest.groovy

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport
+
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.GradleRunner
+import org.gradle.testkit.runner.TaskOutcome
+
+class ResolveTransportVersionConflictFuncTest extends AbstractTransportVersionFuncTest {
+
+    GradleRunner runResolveAndValidateTask() {
+        List<String> args = List.of(":myserver:validateTransportVersionResources", ":myserver:resolveTransportVersionConflict")
+        return gradleRunner(args.toArray())
+    }
+
+    void assertResolveAndValidateSuccess(BuildResult result) {
+        assert result.task(":myserver:resolveTransportVersionConflict").outcome == TaskOutcome.SUCCESS
+        assert result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
+
+    def "update flag works with current"() {
+        given:
+        referableAndReferencedTransportVersion("new_tv", "8123000")
+        file("myserver/src/main/resources/transport/latest/9.2.csv").text =
+            """
+            <<<<<<< HEAD
+            existing_92,8123000
+            =======
+            new_tv,8123000
+            >>>>>> name
+            """.strip()
+
+        when:
+        def result = runResolveAndValidateTask().build()
+
+        then:
+        assertResolveAndValidateSuccess(result)
+        assertReferableDefinition("existing_92", "8123000,8012001")
+        assertReferableDefinition("new_tv", "8124000")
+        assertUpperBound("9.2", "new_tv,8124000")
+    }
+
+    def "update flag works with multiple branches"() {
+        given:
+        referableAndReferencedTransportVersion("new_tv", "8123000,8012001,7123001")
+        file("myserver/src/main/resources/transport/latest/9.2.csv").text =
+            """
+            <<<<<<< HEAD
+            existing_92,8123000
+            =======
+            new_tv,8123000
+            >>>>>> name
+            """.strip()
+        file("myserver/src/main/resources/transport/latest/9.1.csv").text =
+            """
+            <<<<<<< HEAD
+            existing_92,8012001
+            =======
+            new_tv,8012001
+            >>>>>> name
+            """.strip()
+        file("myserver/src/main/resources/transport/latest/8.19.csv").text =
+            """
+            <<<<<<< HEAD
+            initial_8.19.7,7123001
+            =======
+            new_tv,7123001
+            >>>>>> name
+            """.strip()
+
+        when:
+        def result = runResolveAndValidateTask().build()
+
+        then:
+        assertResolveAndValidateSuccess(result)
+        assertReferableDefinition("existing_92", "8123000,8012001")
+        assertUnreferableDefinition("initial_8.19.7", "7123001")
+        assertReferableDefinition("new_tv", "8124000,8012002,7123002")
+        assertUpperBound("9.2", "new_tv,8124000")
+        assertUpperBound("9.1", "new_tv,8012002")
+        assertUpperBound("8.19", "new_tv,7123002")
+    }
+
+    def "no new transport version is idempotent"() {
+        when:
+        def result = runResolveAndValidateTask().build()
+
+        then:
+        assertResolveAndValidateSuccess(result)
+        assertUpperBound("9.2", "existing_92,8123000")
+    }
+}

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy

@@ -19,14 +19,14 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         List<String> args = new ArrayList<>()
         args.add(":myserver:validateTransportVersionResources")
         args.add(":myserver:generateTransportVersion")
-        args.addAll(additionalArgs);
+        args.addAll(additionalArgs)
         return gradleRunner(args.toArray())
     }
 
     def runGenerateTask(String... additionalArgs) {
         List<String> args = new ArrayList<>()
         args.add(":myserver:generateTransportVersion")
-        args.addAll(additionalArgs);
+        args.addAll(additionalArgs)
         return gradleRunner(args.toArray())
     }
 
@@ -90,6 +90,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertUpperBound("9.2", "new_tv,8124000")
     }
 
+    /*
+    temporarily muted, see https://github.com/elastic/elasticsearch/pull/135226
+
     def "invalid changes to a upper bounds should be reverted"() {
         given:
         transportVersionUpperBound("9.2", "modification", "9000000")
@@ -144,7 +147,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertReferableDefinitionDoesNotExist("test_tv")
         assertUpperBound("9.2", "existing_92,8123000")
         assertUpperBound("9.1", "existing_92,8012001")
-    }
+    }*/
 
     def "a reference can be renamed"() {
         given:
@@ -242,8 +245,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
     def "unreferenced definitions are removed"() {
         given:
         referableTransportVersion("test_tv", "8124000,8012002")
+        /*
+        TODO: reset of upper bounds
         transportVersionUpperBound("9.2", "test_tv", "8124000")
         transportVersionUpperBound("9.1", "test_tv", "8012002")
+         */
 
         when:
         def result = runGenerateAndValidateTask().build()
@@ -277,77 +283,6 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertUpperBound("9.2", "second_tv,8124000")
     }
 
-    def "update flag works with current"() {
-        given:
-        referableAndReferencedTransportVersion("new_tv", "8123000")
-        file("myserver/src/main/resources/transport/latest/9.2.csv").text =
-            """
-            <<<<<<< HEAD
-            existing_92,8123000
-            =======
-            new_tv,8123000
-            >>>>>> name
-            """.strip()
-
-        when:
-        def result = runGenerateAndValidateTask("--resolve-conflict").build()
-
-        then:
-        assertGenerateAndValidateSuccess(result)
-        assertReferableDefinition("existing_92", "8123000,8012001")
-        assertReferableDefinition("new_tv", "8124000")
-        assertUpperBound("9.2", "new_tv,8124000")
-    }
-
-    def "update flag works with multiple branches"() {
-        given:
-        referableAndReferencedTransportVersion("new_tv", "8123000,8012001,7123001")
-        file("myserver/src/main/resources/transport/latest/9.2.csv").text =
-            """
-            <<<<<<< HEAD
-            existing_92,8123000
-            =======
-            new_tv,8123000
-            >>>>>> name
-            """.strip()
-        file("myserver/src/main/resources/transport/latest/9.1.csv").text =
-            """
-            <<<<<<< HEAD
-            existing_92,8012001
-            =======
-            new_tv,8012001
-            >>>>>> name
-            """.strip()
-        file("myserver/src/main/resources/transport/latest/8.19.csv").text =
-            """
-            <<<<<<< HEAD
-            initial_8.19.7,7123001
-            =======
-            new_tv,7123001
-            >>>>>> name
-            """.strip()
-
-        when:
-        def result = runGenerateAndValidateTask("--resolve-conflict").build()
-
-        then:
-        assertGenerateAndValidateSuccess(result)
-        assertReferableDefinition("existing_92", "8123000,8012001")
-        assertUnreferableDefinition("initial_8.19.7", "7123001")
-        assertReferableDefinition("new_tv", "8124000,8012002,7123002")
-        assertUpperBound("9.2", "new_tv,8124000")
-        assertUpperBound("9.1", "new_tv,8012002")
-        assertUpperBound("8.19", "new_tv,7123002")
-    }
-
-    def "update flag cannot be used with backport branches"() {
-        when:
-        def result = runGenerateTask("--resolve-conflict", "--backport-branches=9.1").buildAndFail()
-
-        then:
-        assertGenerateFailure(result, "Cannot use --resolve-conflict with --backport-branches")
-    }
-
     def "branches param order does not matter"() {
         given:
         referencedTransportVersion("test_tv")
@@ -477,6 +412,8 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         assertUpperBound("9.2", "new_tv,8124000")
     }
 
+    /*
+    TODO: reset of upper bounds
     def "deleted upper bounds files are restored"() {
         given:
         file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete()
@@ -487,11 +424,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
         then:
         assertGenerateAndValidateSuccess(result)
         assertUpperBound("9.2", "existing_92,8123000")
-    }
+    }*/
 
     def "upper bounds files must exist for backport branches"() {
         when:
-        def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
+        def result = runGenerateTask("--name", "new_tv", "--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
 
         then:
         assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [8.19, 9.0, 9.1, 9.2]")

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -294,4 +294,21 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "only current upper bound validated on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.0'
+            }
+        """
+        referableAndReferencedTransportVersion("some_tv", "8124000,8012004")
+        transportVersionUpperBound("9.1", "some_tv", "8012004")
+
+        when:
+        def result = gradleRunner("validateTransportVersionResources").build()
+
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -113,6 +113,9 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
             String versionProperty = System.getProperty("tests.bwc.main.version");
             // We use this phony version as a placeholder for the real version
             if (distribution.getVersion().equals("0.0.0")) {
+                if (versionProperty == null) {
+                    throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
+                }
                 BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
                     Version.fromString(versionProperty),
                     "main",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -44,10 +44,14 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
 
 import java.util.Collection;
 import java.util.Iterator;
@@ -59,6 +63,7 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
+import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
 
 /**
  * Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
@@ -94,6 +99,7 @@ public RestTestBasePlugin(ProviderFactory providerFactory) {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
+        project.getPluginManager().apply(JvmToolchainsPlugin.class);
         var bwcVersions = loadBuildParams(project).get().getBwcVersions();
 
         // Register integ-test and default distributions
@@ -227,6 +233,17 @@ public Void call(Object... args) {
                     String versionString = version.toString();
                     ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
 
+                    if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
+                        var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+                        var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+                            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+                            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+                        });
+                        task.environment(
+                            "ES_FALLBACK_JAVA_HOME",
+                            fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
+                        );
+                    }
                     task.dependsOn(bwcDistro);
                     registerDistributionInputs(task, bwcDistro);
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -74,11 +74,7 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask
 
     @Input
     @Optional
-    @Option(
-        option = "resolve-conflict",
-        description = "Regenerate the transport version currently being added to upstream to resolve a merge conflict"
-    )
-    public abstract Property<Boolean> getResolveConflict();
+    abstract Property<Boolean> getResolveConflict();
 
     /**
      * The name of the upper bounds file which will be used at runtime on the current branch. Normally
@@ -102,13 +98,14 @@ public void run() throws IOException {
         List<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
         String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
 
-        List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
-        Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
-
         getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
         if (targetDefinitionName.isEmpty()) {
-            resetAllUpperBounds(resources);
+            // TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
+            // resetAllUpperBounds(resources);
         } else {
+            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
+            Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
+
             List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
             // (Re)write the definition file.
             resources.writeDefinition(new TransportVersionDefinition(targetDefinitionName, ids, true));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -19,6 +19,7 @@
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Copy;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
@@ -49,9 +50,9 @@ public void apply(Project project) {
                 Directory transportResources = project.getLayout().getProjectDirectory().dir("src/main/resources/" + resourceRoot);
                 spec.getParameters().getTransportResourcesDirectory().set(transportResources);
                 spec.getParameters().getRootDirectory().set(project.getLayout().getSettingsDirectory().getAsFile());
-                Object upstreamRef = project.findProperty("org.elasticsearch.transport.upstreamRef");
-                if (upstreamRef != null) {
-                    spec.getParameters().getUpstreamRefOverride().set(upstreamRef.toString());
+                Provider<String> upstreamRef = project.getProviders().gradleProperty("org.elasticsearch.transport.upstreamRef");
+                if (upstreamRef.isPresent()) {
+                    spec.getParameters().getUpstreamRefOverride().set(upstreamRef.get());
                 }
             });
 
@@ -120,10 +121,7 @@ public void apply(Project project) {
     }
 
     private static String getResourceRoot(Project project) {
-        var resourceRoot = project.findProperty("org.elasticsearch.transport.resourceRoot");
-        if (resourceRoot == null) {
-            resourceRoot = "transport";
-        }
-        return resourceRoot.toString();
+        Provider<String> resourceRootProperty = project.getProviders().gradleProperty("org.elasticsearch.transport.resourceRoot");
+        return resourceRootProperty.isPresent() ? resourceRootProperty.get() : "transport";
     }
 }