[Build] Rework internal build plugin plugin to work with Isolated Projects

This fixes a general flaw in our build logic where we reach out to configurations of other projects.
This is not best practice and breaks future initiatives like IsolatedProjects that allow parallel configuration
of subprojects.

Author: breskeby

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -20,10 +20,14 @@
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Action;
+import org.gradle.api.GradleException;
+import org.gradle.api.Named;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
+import java.util.HashMap;
 import java.util.Optional;
 
 /**
@@ -47,6 +51,23 @@ public void apply(Project project) {
         project.getConfigurations().getByName("testImplementation").getDependencies().clear();
         var extension = project.getExtensions().getByType(PluginPropertiesExtension.class);
 
+        extension.getExtendedPluginsContainer().whenObjectAdded((Action<Named>) named -> {
+            if (ExtendedPluginInternal.class.isAssignableFrom(named.getClass()) == false) {
+                throw new GradleException(
+                    "Using `extendedPlugins` is not supported for internal plugins. Use `extendedPluginProjects` instead."
+                );
+            }
+        });
+        NamedDomainObjectContainer<ExtendedPluginInternal> container = project.container(
+            ExtendedPluginInternal.class,
+            name -> new ExtendedPluginInternal(name)
+        );
+        container.whenObjectAdded(
+            internal -> extension.getExtendedPluginsContainer().add(internal)
+        );
+
+        HashMap<String, String> value = new HashMap<>();
+        extension.getExtensions().add("extendedPluginProjects", container);
         // We've ported this from multiple build scripts where we see this pattern into
         // an extension method as a first step of consolidation.
         // We might want to port this into a general pattern later on.
@@ -86,11 +107,12 @@ public void doCall() {
             NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
                 .getExtensions()
                 .getByName(TestClustersPlugin.EXTENSION_NAME);
-            p.getExtensions().getByType(PluginPropertiesExtension.class).getExtendedPlugins().forEach(pluginName -> {
+            extension.getExtendedPlugins().forEach(pluginName -> {
                 // Auto add any dependent modules
-                findModulePath(project, pluginName).ifPresent(
-                    path -> testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.module(path))
-                );
+                findModulePath(project, pluginName).ifPresent(path -> {
+                    System.out.println("module name:" + pluginName + " path: '" + path + "'");
+                    testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.module(path));
+                });
             });
         });
     }
@@ -129,4 +151,20 @@ protected static void addNoticeGeneration(final Project project, PluginPropertie
             bundleSpec.from(generateNotice);
         }
     }
+
+    public static class ExtendedPluginInternal extends PluginPropertiesExtension.ExtendedPlugin {
+        String projectPath;
+
+        public ExtendedPluginInternal(String name) {
+            super(name);
+        }
+
+        public String getProjectPath() {
+            return projectPath;
+        }
+
+        public void setProjectPath(String projectPath) {
+            this.projectPath = projectPath;
+        }
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalExtendedPluginProjectsExtension.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
+
+public class InternalExtendedPluginProjectsExtension {
+
+    private final PluginPropertiesExtension pluginPropertiesExtension;
+
+    public InternalExtendedPluginProjectsExtension(PluginPropertiesExtension pluginPropertiesExtension) {
+        this.pluginPropertiesExtension = pluginPropertiesExtension;
+    }
+}

build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java

@@ -9,10 +9,13 @@
 
 package org.elasticsearch.gradle.plugin;
 
+import org.gradle.api.Named;
+import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
 import org.gradle.api.file.CopySpec;
 import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.plugins.BasePluginExtension;
+import org.gradle.api.plugins.ExtensionAware;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 
 import java.io.File;
@@ -23,7 +26,7 @@
  * A container for plugin properties that will be written to the plugin descriptor, for easy
  * manipulation in the gradle DSL.
  */
-public class PluginPropertiesExtension {
+abstract public class PluginPropertiesExtension implements ExtensionAware {
     private String name;
 
     private String version;
@@ -33,7 +36,7 @@ public class PluginPropertiesExtension {
     private String classname;
 
     /** Other plugins this plugin extends through SPI */
-    private List<String> extendedPlugins = new ArrayList<>();
+    private NamedDomainObjectContainer<ExtendedPlugin> extendedPlugins;
 
     private boolean hasNativeController;
 
@@ -57,6 +60,7 @@ public class PluginPropertiesExtension {
 
     public PluginPropertiesExtension(Project project) {
         this.project = project;
+        this.extendedPlugins = project.container(ExtendedPlugin.class);
     }
 
     public String getName() {
@@ -94,6 +98,10 @@ public void setClassname(String classname) {
     }
 
     public List<String> getExtendedPlugins() {
+        return this.extendedPlugins.getNames().stream().toList();
+    }
+
+    public NamedDomainObjectContainer<ExtendedPlugin> getExtendedPluginsContainer() {
         return this.extendedPlugins;
     }
 
@@ -152,7 +160,8 @@ public Project getProject() {
     }
 
     public void setExtendedPlugins(List<String> extendedPlugins) {
-        this.extendedPlugins = extendedPlugins;
+        List<ExtendedPlugin> list = extendedPlugins.stream().map((name) -> new ExtendedPlugin(name)).toList();
+        this.extendedPlugins.addAll(list);
     }
 
     public void setBundleSpec(CopySpec bundleSpec) {
@@ -162,4 +171,19 @@ public void setBundleSpec(CopySpec bundleSpec) {
     public CopySpec getBundleSpec() {
         return bundleSpec;
     }
+
+
+    public static class ExtendedPlugin implements Named {
+
+        private final String name;
+        public ExtendedPlugin(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public String getName() {
+            return name;
+        }
+    }
+
 }

build.gradle

@@ -236,9 +236,8 @@ subprojects { proj ->
 }
 
 allprojects {
-  // We disable this plugin for now till we shaked out the issues we see
-  // e.g. see https://github.com/elastic/elasticsearch/issues/72169
-  // apply plugin:'elasticsearch.internal-test-rerun'
+  apply plugin: 'elasticsearch.eclipse'
+  apply plugin: 'elasticsearch.formatting'
 
   plugins.withType(BaseInternalPluginBuildPlugin).whenPluginAdded {
     project.dependencies {
@@ -249,9 +248,6 @@ allprojects {
 
   ext.bwc_tests_enabled = bwc_tests_enabled
 
-  // eclipse configuration
-  apply plugin: 'elasticsearch.eclipse'
-
   /*
    * Allow accessing com/sun/net/httpserver in projects that have
    * configured forbidden apis to allow it.
@@ -349,35 +345,6 @@ allprojects {
       it.enabled = false
     }
   }
-
-  project.afterEvaluate {
-    // Ensure similar tasks in dependent projects run first. The projectsEvaluated here is
-    // important because, while dependencies.all will pickup future dependencies,
-    // it is not necessarily true that the task exists in both projects at the time
-    // the dependency is added.
-    if (project.path == ':test:framework') {
-      // :test:framework:test cannot run before and after :server:test
-      return
-    }
-    tasks.matching { it.name.equals('integTest') }.configureEach { integTestTask ->
-      integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
-    }
-
-/*    configurations.matching { it.canBeResolved }.all { Configuration configuration ->
-      dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
-        Project upstreamProject = dep.dependencyProject
-        if (project.path != upstreamProject?.path) {
-          for (String taskName : ['test', 'integTest']) {
-            project.tasks.matching { it.name == taskName }.configureEach { task ->
-              task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
-            }
-          }
-        }
-      }
-    }*/
-  }
-
-  apply plugin: 'elasticsearch.formatting'
 }
 
 

libs/entitlement/qa/entitlement-test-plugin/build.gradle

@@ -17,7 +17,11 @@ esplugin {
   name = 'entitlement-test-plugin'
   description = 'A test plugin that invokes methods checked by entitlements'
   classname = 'org.elasticsearch.entitlement.qa.test.EntitlementTestPlugin'
-  extendedPlugins = ['entitled']
+  extendedPluginProjects {
+    entitled {
+      projectPath = ':libs:entitlement:qa:entitled-plugin'
+    }
+  }
 }
 
 dependencies {

modules/aggregations/build.gradle

@@ -13,7 +13,11 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
   description = 'Adds "built in" aggregations to Elasticsearch.'
   classname ='org.elasticsearch.aggregations.AggregationsPlugin'
-  extendedPlugins = ['lang-painless']
+  extendedPluginProjects {
+    "lang-painless" {
+      projectPath = ':modules:lang-painless'
+    }
+  }
 }
 
 restResources {

modules/analysis-common/build.gradle

@@ -15,7 +15,11 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
   description = 'Adds "built in" analyzers to Elasticsearch.'
   classname = 'org.elasticsearch.analysis.common.CommonAnalysisPlugin'
-  extendedPlugins = ['lang-painless']
+  extendedPluginProjects {
+    "lang-painless" {
+      projectPath = ':modules:lang-painless'
+    }
+  }
 }
 
 restResources {

modules/ingest-common/build.gradle

@@ -15,7 +15,11 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 esplugin {
   description = 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
   classname ='org.elasticsearch.ingest.common.IngestCommonPlugin'
-  extendedPlugins = ['lang-painless']
+  extendedPluginProjects {
+    "lang-painless" {
+      projectPath = ':modules:lang-painless'
+    }
+  }
 }
 
 dependencies {

modules/runtime-fields-common/build.gradle

@@ -14,7 +14,11 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test'
 esplugin {
   description = 'Module for runtime fields features and extensions that have large dependencies'
   classname = 'org.elasticsearch.runtimefields.RuntimeFieldsCommonPlugin'
-  extendedPlugins = ['lang-painless']
+  extendedPluginProjects {
+    "lang-painless" {
+      projectPath = ':modules:lang-painless'
+    }
+  }
 }
 
 dependencies {

plugins/examples/painless-whitelist/build.gradle

@@ -13,7 +13,11 @@ esplugin {
   name = 'painless-whitelist'
   description = 'An example whitelisting additional classes and methods in painless'
   classname ='org.elasticsearch.example.painlesswhitelist.MyWhitelistPlugin'
-  extendedPlugins = ['lang-painless']
+  extendedPluginProjects {
+    "lang-painless" {
+      projectPath = ':modules:lang-painless'
+    }
+  }
   licenseFile = rootProject.file('AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt')
   noticeFile = rootProject.file('NOTICE.txt')
 }