Merge remote-tracking branch 'elastic/main' into skip-disi-metric-field

Author: dnhatn

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -334,8 +334,8 @@ steps:
         env:
           BWC_VERSION: 8.18.6
 
-      - label: "{{matrix.image}} / 8.19.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.3
+      - label: "{{matrix.image}} / 8.19.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,7 +348,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.3
+          BWC_VERSION: 8.19.4
 
       - label: "{{matrix.image}} / 9.0.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.6

.buildkite/pipelines/periodic.yml

@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.3#bwcTest
+      - label: 8.19.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.3
+          BWC_VERSION: 8.19.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -18,7 +18,7 @@ BWC_VERSION:
   - "8.16.6"
   - "8.17.10"
   - "8.18.6"
-  - "8.19.3"
+  - "8.19.4"
   - "9.0.6"
   - "9.1.3"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
   - "8.18.6"
-  - "8.19.3"
+  - "8.19.4"
   - "9.0.6"
   - "9.1.3"
   - "9.2.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -330,24 +330,22 @@ private static void configureEntitlements(Project project) {
             .matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
             .configureEach(test -> {
                 // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+                SystemPropertyCommandLineArgumentProvider nonInputSystemProperties = test.getExtensions()
+                    .getByType(SystemPropertyCommandLineArgumentProvider.class);
 
                 // Agent
-                if (agentFiles.isEmpty() == false) {
-                    test.getInputs().files(agentFiles);
-                    test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
-                    test.systemProperty("jdk.attach.allowAttachSelf", true);
-                }
+                test.getInputs().files(agentFiles).optional(true);
+                nonInputSystemProperties.systemProperty("es.entitlement.agentJar", agentFiles::getAsPath);
+                nonInputSystemProperties.systemProperty("jdk.attach.allowAttachSelf", () -> agentFiles.isEmpty() ? "false" : "true");
 
                 // Bridge
-                if (bridgeFiles.isEmpty() == false) {
-                    String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
-                    test.getInputs().files(bridgeFiles);
-                    // Tests may not be modular, but the JDK still is
-                    test.jvmArgs(
-                        "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
-                            + modulesContainingEntitlementInstrumentation
-                    );
-                }
+                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                test.getInputs().files(bridgeFiles).optional(true);
+                // Tests may not be modular, but the JDK still is
+                test.jvmArgs(
+                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                        + modulesContainingEntitlementInstrumentation
+                );
             });
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ProjectSubscribeBuildService.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+public abstract class ProjectSubscribeBuildService implements BuildService<BuildServiceParameters.None> {
+
+    private final ProviderFactory providerFactory;
+
+    /**
+     * The filling of this map depends on the order of #registerProjectForTopic being called.
+     * This is usually done during configuration phase, but we do not enforce yet the time of this method call.
+     * The values are LinkedHashSet to preserve the order of registration mostly to provide a predicatable order
+     * when running consecutive builds.
+     * */
+    private final Map<String, Collection<String>> versionsByTopic = new HashMap<>();
+
+    @Inject
+    public ProjectSubscribeBuildService(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    /**
+     * Returning a provider so the evaluation of the map value is deferred to when the provider is queried.
+     * */
+    public Provider<Collection<String>> getProjectsByTopic(String topic) {
+        return providerFactory.provider(() -> versionsByTopic.computeIfAbsent(topic, k -> new java.util.LinkedHashSet<>()));
+    }
+
+    public void registerProjectForTopic(String topic, Project project) {
+        versionsByTopic.computeIfAbsent(topic, k -> new java.util.LinkedHashSet<>()).add(project.getPath());
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ProjectSubscribeServicePlugin.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+
+/**
+ * This plugin registers a {@link ProjectSubscribeBuildService} to allow projects to
+ * communicate with each other during the configuration phase.
+ *
+ * For example, a project can register itself as a publisher of a topic, and other
+ * projects can resolve projects that have registered as publishers of that topic.
+ *
+ * The actual resolution of whatever data is usually done using dependency declarations.
+ * Be aware the state of the list depends on the order of project configuration and
+ * consuming on configuration phase before task graph calculation phase should be avoided.
+ *
+ * We want to avoid discouraged plugin api usage like project.allprojects or project.subprojects
+ * in plugins to avoid unnecessary configuration of projects and not break project isolation and break
+ * See https://docs.gradle.org/current/userguide/isolated_projects.html
+ * */
+public class ProjectSubscribeServicePlugin implements Plugin<Project> {
+
+    private Provider<ProjectSubscribeBuildService> publishSubscribe;
+
+    @Override
+    public void apply(Project project) {
+        publishSubscribe = project.getGradle().getSharedServices().registerIfAbsent("publishSubscribe", ProjectSubscribeBuildService.class);
+    }
+
+    public Provider<ProjectSubscribeBuildService> getService() {
+        return publishSubscribe;
+    }
+
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java

@@ -44,7 +44,6 @@ public Path getDefinitionsDirectory() {
 
     @TaskAction
     public void generateTransportVersionManifest() throws IOException {
-
         Path definitionsDir = getDefinitionsDirectory();
         Path manifestFile = getManifestFile().get().getAsFile().toPath();
         try (var writer = Files.newBufferedWriter(manifestFile)) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java

@@ -9,19 +9,27 @@
 
 package org.elasticsearch.gradle.internal.transport;
 
+import org.elasticsearch.gradle.internal.ProjectSubscribeServicePlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
-import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
+import static org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin.TRANSPORT_REFERENCES_TOPIC;
+
 public class TransportVersionReferencesPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(LifecycleBasePlugin.class);
 
+        project.getPlugins()
+            .apply(ProjectSubscribeServicePlugin.class)
+            .getService()
+            .get()
+            .registerProjectForTopic(TRANSPORT_REFERENCES_TOPIC, project);
+
         var collectTask = project.getTasks()
             .register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> {
                 t.setGroup("Transport Versions");
@@ -31,9 +39,7 @@ public void apply(Project project) {
                 t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt"));
             });
 
-        Configuration tvReferencesConfig = project.getConfigurations().create("transportVersionReferences", c -> {
-            c.setCanBeConsumed(true);
-            c.setCanBeResolved(false);
+        var tvReferencesConfig = project.getConfigurations().consumable("transportVersionReferences", c -> {
             c.attributes(TransportVersionReference::addArtifactAttribute);
         });
         project.getArtifacts().add(tvReferencesConfig.getName(), collectTask);