Merge branch '9.1' into backport/9.1/pr-130377

Author: pquentin

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -175,6 +175,8 @@ private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<Ma
     private static void configureWithShadowPlugin(Project project, MavenPublication publication) {
         var shadow = project.getExtensions().getByType(ShadowExtension.class);
         shadow.component(publication);
+        publication.artifact(project.getTasks().named("javadocJar"));
+        publication.artifact(project.getTasks().named("sourcesJar"));
     }
 
     private static void addScmInfo(XmlProvider xml, GitInfo gitInfo) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -34,9 +34,11 @@
 import java.io.File;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
+import static java.util.stream.Collectors.joining;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 import static org.elasticsearch.gradle.util.FileUtils.mkdirs;
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
@@ -173,6 +175,16 @@ public void execute(Task t) {
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
 
+            SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+            SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
+            if ("test".equals(test.getName()) && mainSourceSet != null && testSourceSet != null) {
+                FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
+                FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
+                FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
+                test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
+            }
+
             test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get());
             test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get());
 
@@ -205,46 +217,122 @@ public void execute(Task t) {
             }
 
             /*
-             *  If this project builds a shadow JAR than any unit tests should test against that artifact instead of
+             *  If this project builds a shadow JAR then any unit tests should test against that artifact instead of
              *  compiled class output and dependency jars. This better emulates the runtime environment of consumers.
              */
             project.getPluginManager().withPlugin("com.gradleup.shadow", p -> {
                 if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) {
                     // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these
-                    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-                    FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
                     // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR
                     Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.CONFIGURATION_NAME);
                     // Add the shadow JAR artifact itself
                     FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
-                    FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
+                    FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
+                    FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
                     test.setClasspath(testRuntime.minus(mainRuntime).plus(shadowConfig).plus(shadowJar));
                 }
             });
         });
-        configureImmutableCollectionsPatch(project);
+        configureJavaBaseModuleOptions(project);
+        configureEntitlements(project);
+    }
+
+    /**
+     * Computes and sets the {@code --patch-module=java.base} and {@code --add-opens=java.base} JVM command line options.
+     */
+    private void configureJavaBaseModuleOptions(Project project) {
+        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
+            FileCollection patchedImmutableCollections = patchedImmutableCollections(project);
+            if (patchedImmutableCollections != null) {
+                test.getInputs().files(patchedImmutableCollections);
+                test.systemProperty("tests.hackImmutableCollections", "true");
+            }
+
+            FileCollection entitlementBridge = entitlementBridge(project);
+            if (entitlementBridge != null) {
+                test.getInputs().files(entitlementBridge);
+            }
+
+            test.getJvmArgumentProviders().add(() -> {
+                String javaBasePatch = Stream.concat(
+                    singleFilePath(patchedImmutableCollections).map(str -> str + "/java.base"),
+                    singleFilePath(entitlementBridge)
+                ).collect(joining(File.pathSeparator));
+
+                return javaBasePatch.isEmpty()
+                    ? List.of()
+                    : List.of("--patch-module=java.base=" + javaBasePatch, "--add-opens=java.base/java.util=ALL-UNNAMED");
+            });
+        });
     }
 
-    private void configureImmutableCollectionsPatch(Project project) {
+    private Stream<String> singleFilePath(FileCollection collection) {
+        return Stream.ofNullable(collection).filter(fc -> fc.isEmpty() == false).map(FileCollection::getSingleFile).map(File::toString);
+    }
+
+    private static FileCollection patchedImmutableCollections(Project project) {
         String patchProject = ":test:immutable-collections-patch";
         if (project.findProject(patchProject) == null) {
-            return; // build tests may not have this project, just skip
+            return null; // build tests may not have this project, just skip
         }
         String configurationName = "immutableCollectionsPatch";
         FileCollection patchedFileCollection = project.getConfigurations()
             .create(configurationName, config -> config.setCanBeConsumed(false));
         var deps = project.getDependencies();
         deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch")));
-        project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
-            test.getInputs().files(patchedFileCollection);
-            test.systemProperty("tests.hackImmutableCollections", "true");
-            test.getJvmArgumentProviders()
-                .add(
-                    () -> List.of(
-                        "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base",
-                        "--add-opens=java.base/java.util=ALL-UNNAMED"
-                    )
+        return patchedFileCollection;
+    }
+
+    private static FileCollection entitlementBridge(Project project) {
+        return project.getConfigurations().findByName("entitlementBridge");
+    }
+
+    /**
+     * Sets the required JVM options and system properties to enable entitlement enforcement on tests.
+     * <p>
+     * One command line option is set in {@link #configureJavaBaseModuleOptions} out of necessity,
+     * since the command line can have only one {@code --patch-module} option for a given module.
+     */
+    private static void configureEntitlements(Project project) {
+        Configuration agentConfig = project.getConfigurations().create("entitlementAgent");
+        Project agent = project.findProject(":libs:entitlement:agent");
+        if (agent != null) {
+            agentConfig.defaultDependencies(
+                deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:agent"))); }
+            );
+        }
+        FileCollection agentFiles = agentConfig;
+
+        Configuration bridgeConfig = project.getConfigurations().create("entitlementBridge");
+        Project bridge = project.findProject(":libs:entitlement:bridge");
+        if (bridge != null) {
+            bridgeConfig.defaultDependencies(
+                deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:bridge"))); }
+            );
+        }
+        FileCollection bridgeFiles = bridgeConfig;
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+
+            // Agent
+            if (agentFiles.isEmpty() == false) {
+                test.getInputs().files(agentFiles);
+                test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
+                test.systemProperty("jdk.attach.allowAttachSelf", true);
+            }
+
+            // Bridge
+            if (bridgeFiles.isEmpty() == false) {
+                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                test.getInputs().files(bridgeFiles);
+                // Tests may not be modular, but the JDK still is
+                test.jvmArgs(
+                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                        + modulesContainingEntitlementInstrumentation
                 );
+            }
         });
     }
+
 }

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java

@@ -188,7 +188,7 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) {
     }
 
     private String getResource(String name) throws Exception {
-        return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8);
+        return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8).replace("\r", "");
     }
 
     private void writeResource(String name, String contents) throws Exception {

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -18,8 +18,11 @@
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.language.jvm.tasks.ProcessResources;
 
+import java.util.List;
+
 import javax.inject.Inject;
 
 /**
@@ -53,5 +56,11 @@ public void apply(Project project) {
         project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> {
             task.into("META-INF", copy -> copy.from(testBuildInfoTask));
         });
+
+        if (project.getRootProject().getName().equals("elasticsearch")) {
+            project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
+                test.systemProperty("es.entitlement.enableForTests", "true");
+            });
+        }
     }
 }

docs/changelog/127797.yaml

@@ -0,0 +1,5 @@
+pr: 130303
+summary: Drain responses on completion for `TransportNodesAction`
+area: Distributed
+type: bug
+issues: []

docs/changelog/130303.yaml

@@ -0,0 +1,5 @@
+pr: 130421
+summary: Support avg on aggregate metric double
+area: ES|QL
+type: bug
+issues: []

docs/changelog/130421.yaml

@@ -59,8 +59,11 @@ $$$search-throttled$$$`search_throttled`
 `flush`
 :   For [flush](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-flush) and [translog](/reference/elasticsearch/index-settings/translog.md) `fsync` operations. Thread pool type is `scaling` with a keep-alive of `5m` and a default maximum size of `min(5, (`[`# of allocated processors`](#node.processors)`) / 2)`.
 
+`merge`
+:   For [merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-merge.html) operations of all the shards on the node. Thread pool type is `scaling` with a keep-alive of `5m` and a default maximum size of [`# of allocated processors`](#node.processors).
+
 `force_merge`
-:   For [force merge](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-forcemerge) operations. Thread pool type is `fixed` with a size of `max(1, (`[`# of allocated processors`](#node.processors)`) / 8)` and an unbounded queue size.
+:   For waiting on blocking [force merge](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-forcemerge) operations. Thread pool type is `fixed` with a size of `max(1, (`[`# of allocated processors`](#node.processors)`) / 8)` and an unbounded queue size.
 
 `management`
 :   For cluster management. Thread pool type is `scaling` with a keep-alive of `5m` and a default maximum size of `5`.

docs/reference/elasticsearch/configuration-reference/thread-pool-settings.md

@@ -13,10 +13,32 @@ The merge process uses auto-throttling to balance the use of hardware resources
 
 ## Merge scheduling [merge-scheduling]
 
-The merge scheduler (ConcurrentMergeScheduler) controls the execution of merge operations when they are needed. Merges run in separate threads, and when the maximum number of threads is reached, further merges will wait until a merge thread becomes available.
+The merge scheduler controls the execution of merge operations when they are needed.
+Merges run on the dedicated `merge` thread pool.
+Smaller merges are prioritized over larger ones, across all shards on the node.
+Merges are disk IO throttled so that bursts, while merging activity is otherwise low, are smoothed out in order to not impact indexing throughput.
+There is no limit on the number of merges that can be enqueued for execution on the thread pool.
+However, beyond a certain per-shard limit, after merging is completely disk IO un-throttled, indexing for the shard will itself be throttled until merging catches up.
 
-The merge scheduler supports the following *dynamic* setting:
+The available disk space is periodically monitored, such that no new merge tasks are scheduled for execution when the available disk space is low.
+This is in order to prevent that the temporary disk space, which is required while merges are executed, completely fills up the disk space on the node.
+
+The merge scheduler supports the following *dynamic* settings:
 
 `index.merge.scheduler.max_thread_count`
-:   The maximum number of threads on a single shard that may be merging at once. Defaults to `Math.max(1, Math.min(4, <<node.processors, node.processors>> / 2))` which works well for a good solid-state-disk (SSD). If your index is on spinning platter drives instead, decrease this to 1.
+:   The maximum number of threads on a **single** shard that may be merging at once. Defaults to `Math.max(1, Math.min(4, <<node.processors, node.processors>> / 2))` which works well for a good solid-state-disk (SSD). If your index is on spinning platter drives instead, decrease this to 1.
+
+`indices.merge.disk.check_interval`
+:   The time interval for checking the available disk space. Defaults to `5s`.
+
+`indices.merge.disk.watermark.high`
+:   Controls the disk usage watermark, which defaults to `95%`, beyond which no merge tasks can start execution.
+The disk usage tally includes the estimated temporary disk space still required by all the currently executing merge tasks.
+Any merge task scheduled *before* the limit is reached continues execution, even if the limit is exceeded while executing
+(merge tasks are not aborted).
+
+`indices.merge.disk.watermark.high.max_headroom`
+:   Controls the max headroom for the merge disk usage watermark, in case it is specified as percentage or ratio values.
+Defaults to `100GB` when `indices.merge.disk.watermark.high` is not explicitly set.
+This caps the amount of free disk space before merge scheduling is blocked.
 

docs/reference/elasticsearch/index-settings/merge.md

@@ -45,4 +45,14 @@ configure(childProjects.values()) {
      */
     apply plugin: 'elasticsearch.build'
   }
+
+  // This is for any code potentially included in the server at runtime.
+  // Omit oddball libraries that aren't in server.
+  def nonServerLibs = ['plugin-scanner']
+  if (false == nonServerLibs.contains(project.name)) {
+    project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
+      test.systemProperty('es.entitlement.enableForTests', 'true')
+    })
+  }
+
 }