Merge branch 'main' of github.com:elastic/elasticsearch into dvm_update_bbq

Author: tteofili

.buildkite/hooks/pre-command

@@ -96,7 +96,7 @@ fi
 
 if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
   PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
-  PERF_METRICS_USERNAME=$(vault read -field=es_username /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
+  PERF_METRICS_USERNAME=$(vault read -field=es_user /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
   PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/microbenchmarks-metrics)
 
   export PERF_METRICS_HOST

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java

@@ -15,9 +15,13 @@
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.attributes.Category;
+import org.gradle.api.attributes.Usage;
 import org.gradle.api.plugins.JavaPlugin;
 
 public class DependenciesInfoPlugin implements Plugin<Project> {
+
+    public static String USAGE_ATTRIBUTE = "DependenciesInfo";
+
     @Override
     public void apply(final Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
@@ -43,6 +47,9 @@ public void apply(final Project project) {
             )
         );
 
+        dependenciesInfoFilesConfiguration.attributes(
+            attributes -> attributes.attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, USAGE_ATTRIBUTE))
+        );
         project.getArtifacts().add("dependenciesInfoFiles", depsInfo);
 
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java

@@ -24,9 +24,11 @@
 import java.util.HexFormat;
 import java.util.Locale;
 import java.util.function.Function;
+import java.util.jar.Attributes;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.stream.Collectors;
 
 import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES;
@@ -60,6 +62,10 @@ public String toString() {
         }
     }
 
+    public static void patchJar(File inputJar, File outputJar, Collection<PatcherInfo> patchers) {
+        patchJar(inputJar, outputJar, patchers, false);
+    }
+
     /**
      * Patches the classes in the input JAR file, using the collection of patchers. Each patcher specifies a target class (its jar entry
      * name) and the SHA256 digest on the class bytes.
@@ -69,8 +75,11 @@ public String toString() {
      * @param inputFile the JAR file to patch
      * @param outputFile the output (patched) JAR file
      * @param patchers list of patcher info (classes to patch (jar entry name + optional SHA256 digest) and ASM visitor to transform them)
+     * @param unsignJar whether to remove class signatures from the JAR Manifest; set this to true when patching a signed JAR,
+     *                  otherwise the patched classes will fail to load at runtime due to mismatched signatures.
+     *                  @see <a href="https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html">Understanding Signing and Verification</a>
      */
-    public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers) {
+    public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers, boolean unsignJar) {
         var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity()));
         var mismatchedClasses = new ArrayList<MismatchInfo>();
         try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) {
@@ -101,9 +110,23 @@ public static void patchJar(File inputFile, File outputFile, Collection<PatcherI
                         );
                     }
                 } else {
-                    // Read the entry's data and write it to the new JAR
                     try (InputStream is = jarFile.getInputStream(entry)) {
-                        is.transferTo(jos);
+                        if (unsignJar && entryName.equals("META-INF/MANIFEST.MF")) {
+                            var manifest = new Manifest(is);
+                            for (var manifestEntry : manifest.getEntries().entrySet()) {
+                                var nonSignatureAttributes = new Attributes();
+                                for (var attribute : manifestEntry.getValue().entrySet()) {
+                                    if (attribute.getKey().toString().endsWith("Digest") == false) {
+                                        nonSignatureAttributes.put(attribute.getKey(), attribute.getValue());
+                                    }
+                                }
+                                manifestEntry.setValue(nonSignatureAttributes);
+                            }
+                            manifest.write(jos);
+                        } else if (unsignJar == false || entryName.matches("META-INF/.*\\.SF") == false) {
+                            // Read the entry's data and write it to the new JAR
+                            is.transferTo(jos);
+                        }
                     }
                 }
                 jos.closeEntry();

distribution/build.gradle

@@ -31,6 +31,9 @@ configurations {
     attributes {
       attribute(Category.CATEGORY_ATTRIBUTE, project.getObjects().named(Category.class, Category.DOCUMENTATION))
     }
+    attributes {
+      attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, DependenciesInfoPlugin.USAGE_ATTRIBUTE))
+    }
   }
   featuresMetadata {
     attributes {

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmErgonomics.java

@@ -28,6 +28,8 @@
  */
 final class JvmErgonomics {
 
+    static final double DIRECT_MEMORY_TO_HEAP_FACTOR = 0.5;
+
     private JvmErgonomics() {
         throw new AssertionError("No instances intended");
     }
@@ -44,7 +46,7 @@ static List<String> choose(final List<String> userDefinedJvmOptions, Settings no
         final long heapSize = JvmOption.extractMaxHeapSize(finalJvmOptions);
         final long maxDirectMemorySize = JvmOption.extractMaxDirectMemorySize(finalJvmOptions);
         if (maxDirectMemorySize == 0) {
-            ergonomicChoices.add("-XX:MaxDirectMemorySize=" + heapSize / 2);
+            ergonomicChoices.add("-XX:MaxDirectMemorySize=" + (long) (DIRECT_MEMORY_TO_HEAP_FACTOR * heapSize));
         }
 
         final boolean tuneG1GCForSmallHeap = tuneG1GCForSmallHeap(heapSize);

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.node.NodeRoleSettings;
 
 import java.io.IOException;
@@ -37,6 +38,8 @@ public class MachineDependentHeap {
     protected static final long MAX_HEAP_SIZE = GB * 31; // 31GB
     protected static final long MIN_HEAP_SIZE = 1024 * 1024 * 128; // 128MB
 
+    private static final FeatureFlag NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG = new FeatureFlag("new_ml_memory_computation");
+
     public MachineDependentHeap() {}
 
     /**
@@ -76,12 +79,16 @@ protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long av
             /*
              * Machine learning only node.
              *
-             * <p>Heap is computed as:
-             * <ul>
-             *     <li>40% of total system memory when total system memory 16 gigabytes or less.</li>
-             *     <li>40% of the first 16 gigabytes plus 10% of memory above that when total system memory is more than 16 gigabytes.</li>
-             *     <li>The absolute maximum heap size is 31 gigabytes.</li>
-             * </ul>
+             * The memory reserved for Java is computed as:
+             *   - 40% of total system memory when total system memory 16 gigabytes or less.
+             *   - 40% of the first 16 gigabytes plus 10% of memory above that when total system memory is more than 16 gigabytes.
+             *   - The absolute maximum heap size is 31 gigabytes.
+             *
+             * This Java memory is divided as follows:
+             *     - 2/3 of the Java memory is reserved for the Java heap.
+             *     - 1/3 of the Java memory is reserved for the Java direct memory.
+             *
+             * The direct memory being half of the heap is set by the JvmErgonomics class.
              *
              * In all cases the result is rounded down to the next whole multiple of 4 megabytes.
              * The reason for doing this is that Java will round requested heap sizes to a multiple
@@ -95,13 +102,22 @@ protected int getHeapSizeMb(Settings nodeSettings, MachineNodeRole role, long av
              *
              * If this formula is changed then corresponding changes must be made to the {@code NativeMemoryCalculator} and
              * {@code MlAutoscalingDeciderServiceTests} classes in the ML plugin code. Failure to keep the logic synchronized
-             * could result in repeated autoscaling up and down.
+             * could result in ML processes crashing with OOM errors or repeated autoscaling up and down.
              */
             case ML_ONLY -> {
-                if (availableMemory <= (GB * 16)) {
-                    yield mb((long) (availableMemory * .4), 4);
+                double heapFractionBelow16GB = 0.4;
+                double heapFractionAbove16GB = 0.1;
+                if (NEW_ML_MEMORY_COMPUTATION_FEATURE_FLAG.isEnabled()) {
+                    heapFractionBelow16GB = 0.4 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
+                    heapFractionAbove16GB = 0.1 / (1.0 + JvmErgonomics.DIRECT_MEMORY_TO_HEAP_FACTOR);
+                }
+                if (availableMemory <= GB * 16) {
+                    yield mb((long) (availableMemory * heapFractionBelow16GB), 4);
                 } else {
-                    yield mb((long) min((GB * 16) * .4 + (availableMemory - GB * 16) * .1, MAX_HEAP_SIZE), 4);
+                    yield mb(
+                        (long) min(GB * 16 * heapFractionBelow16GB + (availableMemory - GB * 16) * heapFractionAbove16GB, MAX_HEAP_SIZE),
+                        4
+                    );
                 }
             }
             /*

distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java

@@ -56,13 +56,13 @@ public void testMasterOnlyOptions() throws Exception {
     }
 
     public void testMlOnlyOptions() throws Exception {
-        assertHeapOptions(1, containsInAnyOrder("-Xmx408m", "-Xms408m"), "ml");
-        assertHeapOptions(4, containsInAnyOrder("-Xmx1636m", "-Xms1636m"), "ml");
-        assertHeapOptions(32, containsInAnyOrder("-Xmx8192m", "-Xms8192m"), "ml");
-        assertHeapOptions(64, containsInAnyOrder("-Xmx11468m", "-Xms11468m"), "ml");
+        assertHeapOptions(1, containsInAnyOrder("-Xmx272m", "-Xms272m"), "ml");
+        assertHeapOptions(4, containsInAnyOrder("-Xmx1092m", "-Xms1092m"), "ml");
+        assertHeapOptions(32, containsInAnyOrder("-Xmx5460m", "-Xms5460m"), "ml");
+        assertHeapOptions(64, containsInAnyOrder("-Xmx7644m", "-Xms7644m"), "ml");
         // We'd never see a node this big in Cloud, but this assertion proves that the 31GB absolute maximum
         // eventually kicks in (because 0.4 * 16 + 0.1 * (263 - 16) > 31)
-        assertHeapOptions(263, containsInAnyOrder("-Xmx31744m", "-Xms31744m"), "ml");
+        assertHeapOptions(263, containsInAnyOrder("-Xmx21228m", "-Xms21228m"), "ml");
     }
 
     public void testDataNodeOptions() throws Exception {

docs/changelog/128613.yaml

@@ -0,0 +1,5 @@
+pr: 128613
+summary: Improve support for bytecode patching signed jars
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/128742.yaml

@@ -0,0 +1,5 @@
+pr: 128742
+summary: "Account for Java direct memory on machine learning nodes to prevent out-of-memory crashes."
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/129126.yaml

@@ -0,0 +1,6 @@
+pr: 129126
+summary: "Synthetic source: avoid storing multi fields of type text and `match_only_text`\
+  \ by default"
+area: Mapping
+type: bug
+issues: []