Avoid overhead of zip / unzip modules when packaging distribution (#84660) (#84971)

This removes the overhead of zipping up modules that are immediately unzipped again when packaging
the elasticsearch distribution.
We also move some logic for packaging the elasticsearch distribution into a plugin and remove some outdated
overhead dealing with 'meta plugins' when copying modules into a distribution.

Another follow up and related optimization out of scope of this PR is, to also not zip unzip modules declared
for usage in our test cluster setups.

This partially addresses #76726.

Author: breskeby

build-tools-internal/build.gradle

@@ -35,6 +35,10 @@ gradlePlugin {
       id = 'elasticsearch.build'
       implementationClass = 'org.elasticsearch.gradle.internal.BuildPlugin'
     }
+    distro {
+      id = 'elasticsearch.distro'
+      implementationClass = 'org.elasticsearch.gradle.internal.distribution.ElasticsearchDistributionPlugin'
+    }
     distroTest {
       id = 'elasticsearch.distro-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.DistroTestPlugin'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/distibution/ElasticsearchDistributionPluginFuncTest.groovy

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.distibution
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+class ElasticsearchDistributionPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "copied modules are resolved from explodedBundleZip"() {
+        given:
+        moduleSubProject()
+
+        buildFile << """plugins {
+                id 'elasticsearch.distro'
+            }
+            
+            def someCopy = tasks.register('someCopy', Sync) {
+                into 'build/targetDir'
+            }
+            
+            distro.copyModule(someCopy, project(":module"))
+            """
+        when:
+        def result = gradleRunner("someCopy").build()
+
+        then:
+        result.task(":someCopy").outcome == TaskOutcome.SUCCESS
+        file('build/targetDir/modules/some-test-module/some-test-module.jar').exists()
+        file('build/targetDir/modules/some-test-module/plugin-descriptor.properties').exists()
+    }
+
+    private File moduleSubProject() {
+        settingsFile << "include 'module'"
+        file('module/build.gradle') << """
+            plugins {
+                id 'elasticsearch.esplugin'
+            }
+            
+            esplugin {
+                name = 'some-test-module'
+                classname = 'org.acme.never.used.TestPluginClass'
+                description = 'some plugin description'
+            }
+            
+            // for testing purposes only
+            configurations.compileOnly.dependencies.clear()
+        """
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -20,7 +20,6 @@
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.tasks.bundling.Zip;
 
 import java.util.Optional;
 
@@ -106,11 +105,12 @@ Optional<String> findModulePath(Project project, String pluginName) {
     protected static void addNoticeGeneration(final Project project, PluginPropertiesExtension extension) {
         final var licenseFile = extension.getLicenseFile();
         var tasks = project.getTasks();
+        var bundleSpec = extension.getBundleSpec();
         if (licenseFile != null) {
-            tasks.withType(Zip.class).named("bundlePlugin").configure(zip -> zip.from(licenseFile.getParentFile(), copySpec -> {
-                copySpec.include(licenseFile.getName());
-                copySpec.rename(s -> "LICENSE.txt");
-            }));
+            bundleSpec.from(licenseFile.getParentFile(), s -> {
+                s.include(licenseFile.getName());
+                s.rename(f -> "LICENSE.txt");
+            });
         }
 
         final var noticeFile = extension.getNoticeFile();
@@ -119,7 +119,7 @@ protected static void addNoticeGeneration(final Project project, PluginPropertie
                 noticeTask.setInputFile(noticeFile);
                 noticeTask.source(Util.getJavaMainSourceSet(project).get().getAllJava());
             });
-            tasks.withType(Zip.class).named("bundlePlugin").configure(task -> task.from(generateNotice));
+            bundleSpec.from(generateNotice);
         }
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionExtension.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.tasks.AbstractCopyTask;
+import org.gradle.api.tasks.TaskProvider;
+
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.regex.Pattern;
+
+import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.EXPLODED_BUNDLE_CONFIG;
+
+public class ElasticsearchDistributionExtension {
+    private static final Pattern CONFIG_BIN_REGEX_PATTERN = Pattern.compile("([^\\/]+\\/)?(config|bin)\\/.*");
+
+    private final Project project;
+
+    public ElasticsearchDistributionExtension(Project project) {
+        this.project = project;
+    }
+
+    private Configuration moduleZip(Project module) {
+        var moduleConfigurationCoords = Map.of("path", module.getPath(), "configuration", EXPLODED_BUNDLE_CONFIG);
+        var dep = project.getDependencies().project(moduleConfigurationCoords);
+        return project.getConfigurations().detachedConfiguration(dep);
+    }
+
+    public void copyModule(TaskProvider<AbstractCopyTask> copyTask, Project module) {
+        copyTask.configure(sync -> {
+            var moduleConfig = moduleZip(module);
+            sync.dependsOn(moduleConfig);
+            Callable<File> callableSingleFile = () -> moduleConfig.getSingleFile();
+            sync.from(callableSingleFile, spec -> {
+                spec.setIncludeEmptyDirs(false);
+                // these are handled separately in the log4j config tasks in the :distribution plugin
+                spec.exclude("*/config/log4j2.properties");
+                spec.exclude("config/log4j2.properties");
+
+                // This adds a implicit dependency for 'module' to PluginBuildPlugin which is fine as we just fail
+                // in case an invalid 'module' not applying this plugin is passed here
+                var moduleName = module.getExtensions().getByType(PluginPropertiesExtension.class).getName();
+
+                spec.eachFile(d -> {
+                    if (CONFIG_BIN_REGEX_PATTERN.matcher(d.getRelativePath().getPathString()).matches() == false) {
+                        d.setRelativePath(d.getRelativePath().prepend("modules", moduleName));
+                    }
+                });
+            });
+        });
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/ElasticsearchDistributionPlugin.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+
+/**
+ * Provides an extension named 'distro' to provide common operations used when setting up and bundling
+ * elasticsearch distributions. Having those here instead of in the build scripts makes testing and maintaining
+ * this common functionality easier
+ * */
+public class ElasticsearchDistributionPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        project.getExtensions().create("distro", ElasticsearchDistributionExtension.class, project);
+    }
+}

build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.plugin
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+class PluginBuildPluginFuncTest extends AbstractGradleFuncTest {
+
+    def "can assemble plugin via #taskName"() {
+        given:
+        buildFile << """plugins {
+                id 'elasticsearch.esplugin'
+            }
+            
+            esplugin { 
+                description = 'test plugin'
+                classname = 'com.acme.plugin.TestPlugin'
+            }
+            
+            // for testing purposes only
+            configurations.compileOnly.dependencies.clear()
+            """
+
+        when:
+        def result = gradleRunner(taskName).build()
+
+        then:
+        result.task(taskName).outcome == TaskOutcome.SUCCESS
+        file(expectedOutputPath).exists()
+
+        where:
+        expectedOutputPath                    | taskName
+        "build/distributions/hello-world.zip" | ":bundlePlugin"
+        "build/explodedBundle/"               | ":explodedBundlePlugin"
+
+    }
+
+    def "can resolve plugin as directory without intermediate zipping "() {
+        given:
+        buildFile << """plugins {
+                id 'elasticsearch.esplugin'
+            }
+            
+            esplugin { 
+                name = 'sample-plugin'
+                description = 'test plugin'
+                classname = 'com.acme.plugin.TestPlugin'
+            }
+            
+            // for testing purposes only
+            configurations.compileOnly.dependencies.clear()
+            """
+
+        file('settings.gradle') << "include 'module-consumer'"
+        file('module-consumer/build.gradle') << """
+            configurations {
+                consume
+            }
+            
+            dependencies {
+                consume project(path:':', configuration:'${PluginBuildPlugin.EXPLODED_BUNDLE_CONFIG}')
+            }
+            
+            tasks.register("resolveModule", Copy) {
+                from configurations.consume
+                into "build/resolved"
+            }
+        """
+        when:
+        def result = gradleRunner(":module-consumer:resolveModule").build()
+
+        then:
+        result.task(":module-consumer:resolveModule").outcome == TaskOutcome.SUCCESS
+        result.task(":explodedBundlePlugin").outcome == TaskOutcome.SUCCESS
+        file("module-consumer/build/resolved/sample-plugin.jar").exists()
+        file("module-consumer/build/resolved/plugin-descriptor.properties").exists()
+    }
+}