Merge branch 'main' into ES-12183

Author: alexey-ivanov-es

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -334,8 +334,8 @@ steps:
         env:
           BWC_VERSION: 8.18.6
 
-      - label: "{{matrix.image}} / 8.19.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.3
+      - label: "{{matrix.image}} / 8.19.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,7 +348,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.3
+          BWC_VERSION: 8.19.4
 
       - label: "{{matrix.image}} / 9.0.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.6

.buildkite/pipelines/periodic.yml

@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.3#bwcTest
+      - label: 8.19.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.3
+          BWC_VERSION: 8.19.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -18,7 +18,7 @@ BWC_VERSION:
   - "8.16.6"
   - "8.17.10"
   - "8.18.6"
-  - "8.19.3"
+  - "8.19.4"
   - "9.0.6"
   - "9.1.3"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
   - "8.18.6"
-  - "8.19.3"
+  - "8.19.4"
   - "9.0.6"
   - "9.1.3"
   - "9.2.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -330,24 +330,22 @@ private static void configureEntitlements(Project project) {
             .matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
             .configureEach(test -> {
                 // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+                SystemPropertyCommandLineArgumentProvider nonInputSystemProperties = test.getExtensions()
+                    .getByType(SystemPropertyCommandLineArgumentProvider.class);
 
                 // Agent
-                if (agentFiles.isEmpty() == false) {
-                    test.getInputs().files(agentFiles);
-                    test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
-                    test.systemProperty("jdk.attach.allowAttachSelf", true);
-                }
+                test.getInputs().files(agentFiles).optional(true);
+                nonInputSystemProperties.systemProperty("es.entitlement.agentJar", agentFiles::getAsPath);
+                nonInputSystemProperties.systemProperty("jdk.attach.allowAttachSelf", () -> agentFiles.isEmpty() ? "false" : "true");
 
                 // Bridge
-                if (bridgeFiles.isEmpty() == false) {
-                    String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
-                    test.getInputs().files(bridgeFiles);
-                    // Tests may not be modular, but the JDK still is
-                    test.jvmArgs(
-                        "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
-                            + modulesContainingEntitlementInstrumentation
-                    );
-                }
+                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                test.getInputs().files(bridgeFiles).optional(true);
+                // Tests may not be modular, but the JDK still is
+                test.jvmArgs(
+                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                        + modulesContainingEntitlementInstrumentation
+                );
             });
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ProjectSubscribeBuildService.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+public abstract class ProjectSubscribeBuildService implements BuildService<BuildServiceParameters.None> {
+
+    private final ProviderFactory providerFactory;
+
+    /**
+     * The filling of this map depends on the order of #registerProjectForTopic being called.
+     * This is usually done during configuration phase, but we do not enforce yet the time of this method call.
+     * The values are LinkedHashSet to preserve the order of registration mostly to provide a predicatable order
+     * when running consecutive builds.
+     * */
+    private final Map<String, Collection<String>> versionsByTopic = new HashMap<>();
+
+    @Inject
+    public ProjectSubscribeBuildService(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    /**
+     * Returning a provider so the evaluation of the map value is deferred to when the provider is queried.
+     * */
+    public Provider<Collection<String>> getProjectsByTopic(String topic) {
+        return providerFactory.provider(() -> versionsByTopic.computeIfAbsent(topic, k -> new java.util.LinkedHashSet<>()));
+    }
+
+    public void registerProjectForTopic(String topic, Project project) {
+        versionsByTopic.computeIfAbsent(topic, k -> new java.util.LinkedHashSet<>()).add(project.getPath());
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ProjectSubscribeServicePlugin.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+
+/**
+ * This plugin registers a {@link ProjectSubscribeBuildService} to allow projects to
+ * communicate with each other during the configuration phase.
+ *
+ * For example, a project can register itself as a publisher of a topic, and other
+ * projects can resolve projects that have registered as publishers of that topic.
+ *
+ * The actual resolution of whatever data is usually done using dependency declarations.
+ * Be aware the state of the list depends on the order of project configuration and
+ * consuming on configuration phase before task graph calculation phase should be avoided.
+ *
+ * We want to avoid discouraged plugin api usage like project.allprojects or project.subprojects
+ * in plugins to avoid unnecessary configuration of projects and not break project isolation and break
+ * See https://docs.gradle.org/current/userguide/isolated_projects.html
+ * */
+public class ProjectSubscribeServicePlugin implements Plugin<Project> {
+
+    private Provider<ProjectSubscribeBuildService> publishSubscribe;
+
+    @Override
+    public void apply(Project project) {
+        publishSubscribe = project.getGradle().getSharedServices().registerIfAbsent("publishSubscribe", ProjectSubscribeBuildService.class);
+    }
+
+    public Provider<ProjectSubscribeBuildService> getService() {
+        return publishSubscribe;
+    }
+
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java

@@ -44,7 +44,6 @@ public Path getDefinitionsDirectory() {
 
     @TaskAction
     public void generateTransportVersionManifest() throws IOException {
-
         Path definitionsDir = getDefinitionsDirectory();
         Path manifestFile = getManifestFile().get().getAsFile().toPath();
         try (var writer = Files.newBufferedWriter(manifestFile)) {