Merge branch '8.19' into backport/8.19/pr-140979

Author: rjernst

BUILDING.md

@@ -345,6 +345,12 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
+```bash
+
+# write verification metadata and run the precommit task
+./gradlew --write-verification-metadata sha256 precommit
+```
+
 > [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -53,9 +53,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:jira");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:pagerduty");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java

@@ -68,22 +68,30 @@ public CacheTestFixtureWorkAction() {}
         public void execute() {
             final URLClassLoader urlLoader = createClassLoader(getParameters().getClasspath());
             try {
+                Set<URL> packageUrls = new LinkedHashSet<>();
+                packageUrls.addAll(ClasspathHelper.forPackage("org.elasticsearch.test.fixtures"));
+                packageUrls.addAll(ClasspathHelper.forPackage("co.elastic.elasticsearch.test.fixtures"));
                 Reflections reflections = new Reflections(
-                    new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.elasticsearch.test.fixtures"))
-                        .setScanners(new SubTypesScanner())
+                    new ConfigurationBuilder().setUrls(packageUrls).setScanners(new SubTypesScanner())
                 );
 
                 Class<?> ifClass = Class.forName("org.elasticsearch.test.fixtures.CacheableTestFixture");
                 Set<Class<?>> classes = (Set<Class<?>>) reflections.getSubTypesOf(ifClass);
 
+                System.out.println("Fixtures found: " + classes.size());
                 for (Class<?> cacheableTestFixtureClazz : classes) {
                     if (Modifier.isAbstract(cacheableTestFixtureClazz.getModifiers()) == false) {
-                        Constructor<?> declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor();
-                        declaredConstructor.setAccessible(true);
-                        Object o = declaredConstructor.newInstance();
-                        Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
-                        System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName());
-                        cacheMethod.invoke(o);
+                        try {
+                            Constructor<?> declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor();
+                            declaredConstructor.setAccessible(true);
+                            Object o = declaredConstructor.newInstance();
+                            Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
+                            System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName());
+                            cacheMethod.invoke(o);
+                        } catch (NoSuchMethodException e) {
+                            // Skip classes without a no-arg constructor - they are not meant to be cached directly
+                            System.out.println("Skipping " + cacheableTestFixtureClazz.getName() + " (no no-arg constructor)");
+                        }
                     }
                 }
             } catch (Exception e) {

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:bf732027be3f7f5ecc5fd5334cf5d12c67d5f2a71ebf4230309d0e34b0cb47ad AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:bf732027be3f7f5ecc5fd5334cf5d12c67d5f2a71ebf4230309d0e34b0cb47ad
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:11d66ae7ec7ca1d5a7289750bdd96eb3d8eb592e5b7377f1fc8e4fb556fa7383
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:a42fd0f8865a04b8ab4ff12e9ecbba681ebae4112b9e7e9e34ce9b08ebfa650d AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -68,7 +68,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:a42fd0f8865a04b8ab4ff12e9ecbba681ebae4112b9e7e9e34ce9b08ebfa650d
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b5a03b6a754fa2f9a29e44e316a6c4df1f606cd8a3cd8810ce3d6a3a3134428b
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmArgumentParsingSystemMemoryInfo.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.server.cli;
+
+import java.util.List;
+
+public abstract class JvmArgumentParsingSystemMemoryInfo implements SystemMemoryInfo {
+    private final List<String> userDefinedJvmOptions;
+
+    public JvmArgumentParsingSystemMemoryInfo(List<String> userDefinedJvmOptions) {
+        this.userDefinedJvmOptions = userDefinedJvmOptions;
+    }
+
+    protected long getBytesFromSystemProperty(String systemProperty, long defaultValue) {
+        return userDefinedJvmOptions.stream()
+            .filter(option -> option.startsWith("-D" + systemProperty + "="))
+            .map(totalMemoryOverheadBytesOption -> {
+                try {
+                    long bytes = Long.parseLong(totalMemoryOverheadBytesOption.split("=", 2)[1]);
+                    if (bytes < 0) {
+                        throw new IllegalArgumentException("Negative bytes size specified in [" + totalMemoryOverheadBytesOption + "]");
+                    }
+                    return bytes;
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryOverheadBytesOption + "]", e);
+                }
+            })
+            .reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
+            .orElse(defaultValue);
+    }
+}

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java

@@ -142,7 +142,10 @@ private List<String> jvmOptions(
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
-        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo());
+        final SystemMemoryInfo memoryInfo = new OverheadSystemMemoryInfo(
+            substitutedJvmOptions,
+            new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
+        );
         substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(args.nodeSettings(), memoryInfo, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions, args.nodeSettings());
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops);

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverheadSystemMemoryInfo.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.server.cli;
+
+import java.util.List;
+
+/**
+ * A {@link SystemMemoryInfo} implementation that reduces the reported available system memory by a set amount. This is intended to account
+ * for overhead cost of other bundled Elasticsearch processes, such as the CLI tool launcher. By default, this is
+ * {@link org.elasticsearch.server.cli.OverheadSystemMemoryInfo#SERVER_CLI_OVERHEAD } but can be overridden via the
+ * {@code es.total_memory_overhead_bytes} system property.
+ */
+public class OverheadSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
+    static final long SERVER_CLI_OVERHEAD = 100 * 1024L * 1024L;
+
+    private final SystemMemoryInfo delegate;
+
+    public OverheadSystemMemoryInfo(List<String> userDefinedJvmOptions, SystemMemoryInfo delegate) {
+        super(userDefinedJvmOptions);
+        this.delegate = delegate;
+    }
+
+    @Override
+    public long availableSystemMemory() {
+        long overheadBytes = getBytesFromSystemProperty("es.total_memory_overhead_bytes", SERVER_CLI_OVERHEAD);
+        return delegate.availableSystemMemory() - overheadBytes;
+    }
+}

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/OverridableSystemMemoryInfo.java

@@ -17,33 +17,17 @@
  * has been specified using the {@code es.total_memory_bytes} system property, or
  * else returns the value provided by a fallback provider.
  */
-public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
+public final class OverridableSystemMemoryInfo extends JvmArgumentParsingSystemMemoryInfo {
 
-    private final List<String> userDefinedJvmOptions;
     private final SystemMemoryInfo fallbackSystemMemoryInfo;
 
     public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
-        this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
+        super(userDefinedJvmOptions);
         this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
     }
 
     @Override
     public long availableSystemMemory() {
-
-        return userDefinedJvmOptions.stream()
-            .filter(option -> option.startsWith("-Des.total_memory_bytes="))
-            .map(totalMemoryBytesOption -> {
-                try {
-                    long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
-                    if (bytes < 0) {
-                        throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
-                    }
-                    return bytes;
-                } catch (NumberFormatException e) {
-                    throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
-                }
-            })
-            .reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
-            .orElse(fallbackSystemMemoryInfo.availableSystemMemory());
+        return getBytesFromSystemProperty("es.total_memory_bytes", fallbackSystemMemoryInfo.availableSystemMemory());
     }
 }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -118,6 +118,8 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
             return;
         }
 
+        // Call the GC to try and free up as much heap as we can since we don't intend to do much if any more allocation after this
+        System.gc();
         // we are running in the foreground, so wait for the server to exit
         int exitCode = server.waitFor();
         onExit(exitCode);