Merge branch 'main' into esql/reuse-outputset

Author: costin

build-conventions/settings.gradle

@@ -8,7 +8,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.18.1"
+  id "com.gradle.develocity" version "3.19.2"
 }
 
 rootProject.name = 'build-conventions'

build-tools-internal/settings.gradle

@@ -9,7 +9,7 @@ pluginManagement {
 }
 
 plugins {
-  id "com.gradle.develocity" version "3.18.1"
+  id "com.gradle.develocity" version "3.19.2"
 }
 
 dependencyResolutionManagement {

build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle

@@ -47,7 +47,7 @@ plugins.withType(InternalJavaRestTestPlugin) {
   tasks.withType(StandaloneRestIntegTestTask).configureEach {
     testClassesDirs = sourceSets.javaRestTest.output.classesDirs
     classpath = sourceSets.javaRestTest.runtimeClasspath
-    usesDefaultDistribution()
+    usesDefaultDistribution("BWC tests require full distribution for now")
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -17,10 +17,7 @@
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
@@ -81,29 +78,6 @@ public void doCall() {
         if (isModule == false || isXPackModule) {
             addNoticeGeneration(project, extension);
         }
-        project.afterEvaluate(p -> {
-            @SuppressWarnings("unchecked")
-            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-                .getExtensions()
-                .getByName(TestClustersPlugin.EXTENSION_NAME);
-            p.getExtensions().getByType(PluginPropertiesExtension.class).getExtendedPlugins().forEach(pluginName -> {
-                // Auto add any dependent modules
-                findModulePath(project, pluginName).ifPresent(
-                    path -> testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.module(path))
-                );
-            });
-        });
-    }
-
-    Optional<String> findModulePath(Project project, String pluginName) {
-        return project.getRootProject()
-            .getAllprojects()
-            .stream()
-            .filter(p -> GradleUtils.isModuleProject(p.getPath()))
-            .filter(p -> p.getPlugins().hasPlugin(PluginBuildPlugin.class))
-            .filter(p -> p.getExtensions().getByType(PluginPropertiesExtension.class).getName().equals(pluginName))
-            .findFirst()
-            .map(Project::getPath);
     }
 
     /**

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java

@@ -19,6 +19,8 @@
 import org.gradle.api.Task;
 import org.gradle.api.file.ArchiveOperations;
 import org.gradle.api.plugins.BasePlugin;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.TaskProvider;
 
@@ -103,22 +105,26 @@ private static TaskProvider<Task> registerCheckMlCppNoticeTask(
     ) {
         TaskProvider<Task> checkMlCppNoticeTask = project.getTasks().register("checkMlCppNotice", task -> {
             task.dependsOn(checkExtraction);
+            final Provider<Path> noticePath = checkExtraction.map(
+                c -> c.getDestinationDir()
+                    .toPath()
+                    .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt")
+            );
+            ListProperty<String> expectedMlLicenses = extension.expectedMlLicenses;
             task.doLast(new Action<Task>() {
                 @Override
                 public void execute(Task task) {
                     // this is just a small sample from the C++ notices,
                     // the idea being that if we've added these lines we've probably added all the required lines
-                    final List<String> expectedLines = extension.expectedMlLicenses.get();
-                    final Path noticePath = checkExtraction.get()
-                        .getDestinationDir()
-                        .toPath()
-                        .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/modules/x-pack-ml/NOTICE.txt");
+                    final List<String> expectedLines = expectedMlLicenses.get();
                     final List<String> actualLines;
                     try {
-                        actualLines = Files.readAllLines(noticePath);
+                        actualLines = Files.readAllLines(noticePath.get());
                         for (final String expectedLine : expectedLines) {
                             if (actualLines.contains(expectedLine) == false) {
-                                throw new GradleException("expected [" + noticePath + " to contain [" + expectedLine + "] but it did not");
+                                throw new GradleException(
+                                    "expected [" + noticePath.get() + " to contain [" + expectedLine + "] but it did not"
+                                );
                             }
                         }
                     } catch (IOException ioException) {
@@ -133,43 +139,37 @@ public void execute(Task task) {
     private TaskProvider<Task> registerCheckNoticeTask(Project project, TaskProvider<Copy> checkExtraction) {
         return project.getTasks().register("checkNotice", task -> {
             task.dependsOn(checkExtraction);
-            task.doLast(new Action<Task>() {
-                @Override
-                public void execute(Task task) {
-                    final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2024 Elasticsearch");
-                    final Path noticePath = checkExtraction.get()
-                        .getDestinationDir()
-                        .toPath()
-                        .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt");
-                    assertLinesInFile(noticePath, noticeLines);
-                }
+            var noticePath = checkExtraction.map(
+                copy -> copy.getDestinationDir().toPath().resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/NOTICE.txt")
+            );
+            task.doLast(t -> {
+                final List<String> noticeLines = Arrays.asList("Elasticsearch", "Copyright 2009-2024 Elasticsearch");
+                assertLinesInFile(noticePath.get(), noticeLines);
             });
         });
     }
 
     private TaskProvider<Task> registerCheckLicenseTask(Project project, TaskProvider<Copy> checkExtraction) {
         TaskProvider<Task> checkLicense = project.getTasks().register("checkLicense", task -> {
             task.dependsOn(checkExtraction);
-            task.doLast(new Action<Task>() {
-                @Override
-                public void execute(Task task) {
-                    String licenseFilename = null;
-                    if (project.getName().contains("oss-") || project.getName().equals("integ-test-zip")) {
-                        licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt";
-                    } else {
-                        licenseFilename = "ELASTIC-LICENSE-2.0.txt";
-                    }
-                    final List<String> licenseLines;
-                    try {
-                        licenseLines = Files.readAllLines(project.getRootDir().toPath().resolve("licenses/" + licenseFilename));
-                        final Path licensePath = checkExtraction.get()
-                            .getDestinationDir()
-                            .toPath()
-                            .resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt");
-                        assertLinesInFile(licensePath, licenseLines);
-                    } catch (IOException ioException) {
-                        ioException.printStackTrace();
-                    }
+            String projectName = project.getName();
+            Provider<Path> licensePathProvider = checkExtraction.map(
+                copy -> copy.getDestinationDir().toPath().resolve("elasticsearch-" + VersionProperties.getElasticsearch() + "/LICENSE.txt")
+            );
+            File rootDir = project.getLayout().getSettingsDirectory().getAsFile();
+            task.doLast(t -> {
+                String licenseFilename = null;
+                if (projectName.contains("oss-") || projectName.equals("integ-test-zip")) {
+                    licenseFilename = "AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt";
+                } else {
+                    licenseFilename = "ELASTIC-LICENSE-2.0.txt";
+                }
+                final List<String> licenseLines;
+                try {
+                    licenseLines = Files.readAllLines(rootDir.toPath().resolve("licenses/" + licenseFilename));
+                    assertLinesInFile(licensePathProvider.get(), licenseLines);
+                } catch (IOException ioException) {
+                    ioException.printStackTrace();
                 }
             });
         });

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -49,8 +49,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-annotated-text");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-murmur3");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:repository-hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb");
         map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster");
@@ -74,11 +72,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-constant-keyword");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-unsigned-long");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-version");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:wildcard");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -198,6 +198,11 @@ public void apply(Project project) {
             task.getExtensions().getExtraProperties().set("usesDefaultDistribution", new Closure<Void>(task) {
                 @Override
                 public Void call(Object... args) {
+                    if (reasonForUsageProvided(args) == false) {
+                        throw new IllegalArgumentException(
+                            "Reason for using `usesDefaultDistribution` required.\nUse usesDefaultDistribution(\"reason why default distro is required here\")."
+                        );
+                    }
                     task.dependsOn(defaultDistro);
                     registerDistributionInputs(task, defaultDistro);
 
@@ -212,6 +217,10 @@ public Void call(Object... args) {
 
                     return null;
                 }
+
+                private static boolean reasonForUsageProvided(Object[] args) {
+                    return args.length == 1 && args[0] instanceof String && ((String) args[0]).isBlank() == false;
+                }
             });
 
             // Add `usesBwcDistribution(version)` extension method to test tasks to indicate they require a BWC distribution

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/ArchivedOracleJdkToolchainResolver.java

@@ -29,7 +29,7 @@
  */
 public abstract class ArchivedOracleJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
-    private static final Map<Integer, String> ARCHIVED_BASE_VERSIONS = Maps.of(20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1");
+    private static final Map<Integer, String> ARCHIVED_BASE_VERSIONS = Maps.of(21, "21.0.6", 20, "20.0.2", 19, "19.0.2", 18, "18.0.2.1");
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -58,10 +58,9 @@ public String url(String os, String arch, String extension) {
         }
     }
 
-    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion) implements JdkBuild {
+    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String buildNumber) implements JdkBuild {
         @Override
         public String url(String os, String arch, String extension) {
-            String buildNumber = resolveBuildNumber(languageVersion.asInt());
             return "https://download.java.net/java/early_access/jdk"
                 + languageVersion.asInt()
                 + "/"
@@ -77,29 +76,6 @@ public String url(String os, String arch, String extension) {
                 + "_bin."
                 + extension;
         }
-
-        private static String resolveBuildNumber(int version) {
-            String buildNumber = System.getProperty("runtime.java." + version + ".build");
-            if (buildNumber != null) {
-                System.out.println("buildNumber = " + buildNumber);
-                return buildNumber;
-            }
-            buildNumber = System.getProperty("runtime.java.build");
-            if (buildNumber != null) {
-                System.out.println("buildNumber2 = " + buildNumber);
-                return buildNumber;
-            }
-
-            switch (version) {
-                case 24:
-                    // latest explicitly found build number for 24
-                    return "29";
-                case 25:
-                    return "3";
-                default:
-                    throw new IllegalArgumentException("Unsupported version " + version);
-            }
-        }
     }
 
     private static final Pattern VERSION_PATTERN = Pattern.compile(
@@ -114,15 +90,20 @@ private static String resolveBuildNumber(int version) {
 
     // package private so it can be replaced by tests
     List<JdkBuild> builds = List.of(
-        getBundledJdkBuild(),
-        // release candidate of JDK 24
-        new ReleaseJdkBuild(JavaLanguageVersion.of(24), "download.java.net", "24", "36", "1f9ff9062db4449d8ca828c504ffae90"),
-        new EarlyAccessJdkBuild(JavaLanguageVersion.of(25))
+        getBundledJdkBuild(VersionProperties.getBundledJdkVersion(), VersionProperties.getBundledJdkMajorVersion()),
+        getEarlyAccessBuild(JavaLanguageVersion.of(25), "3")
     );
 
-    private JdkBuild getBundledJdkBuild() {
-        String bundledJdkVersion = VersionProperties.getBundledJdkVersion();
-        JavaLanguageVersion bundledJdkMajorVersion = JavaLanguageVersion.of(VersionProperties.getBundledJdkMajorVersion());
+    static EarlyAccessJdkBuild getEarlyAccessBuild(JavaLanguageVersion languageVersion, String buildNumber) {
+        // first try the unversioned override, then the versioned override which has higher precedence
+        buildNumber = System.getProperty("runtime.java.build", buildNumber);
+        buildNumber = System.getProperty("runtime.java." + languageVersion.asInt() + ".build", buildNumber);
+
+        return new EarlyAccessJdkBuild(languageVersion, buildNumber);
+    }
+
+    static JdkBuild getBundledJdkBuild(String bundledJdkVersion, String bundledJkdMajorVersionString) {
+        JavaLanguageVersion bundledJdkMajorVersion = JavaLanguageVersion.of(bundledJkdMajorVersionString);
         Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(bundledJdkVersion);
         if (jdkVersionMatcher.matches() == false) {
             throw new IllegalStateException("Unable to parse bundled JDK version " + bundledJdkVersion);