Merge branch 'main' into fix-missing-index-handling

Author: smalyshev

.buildkite/scripts/dra-workflow.sh

@@ -70,6 +70,7 @@ echo --- Building release artifacts
   $VERSION_QUALIFIER_ARG \
   buildReleaseArtifacts \
   exportCompressedDockerImages \
+  exportDockerContexts \
   :distribution:generateDependenciesReport
 
 PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script

.ci/scripts/packaging-test.sh

@@ -3,7 +3,7 @@
 # opensuse 15 has a missing dep for systemd
 
 if which zypper > /dev/null ; then
-    sudo zypper install -y insserv-compat
+    sudo zypper install -y insserv-compat docker-buildx
 fi
 
 if [ -e /etc/sysctl.d/99-gce.conf ]; then

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -14,7 +14,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.GlobalRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -126,19 +129,20 @@ public void setUp() throws Exception {
             Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build()
         );
 
-        Metadata.Builder mb = Metadata.builder();
+        final ProjectId projectId = ProjectId.DEFAULT;
+        ProjectMetadata.Builder pmb = ProjectMetadata.builder(projectId);
         for (int i = 1; i <= numIndices; i++) {
-            mb.put(
+            pmb.put(
                 IndexMetadata.builder("test_" + i)
                     .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
                     .numberOfShards(numShards)
                     .numberOfReplicas(numReplicas)
             );
         }
-        Metadata metadata = mb.build();
+        Metadata metadata = Metadata.builder().put(pmb).build();
         RoutingTable.Builder rb = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 1; i <= numIndices; i++) {
-            rb.addAsNew(metadata.getProject().index("test_" + i));
+            rb.addAsNew(metadata.getProject(projectId).index("test_" + i));
         }
         RoutingTable routingTable = rb.build();
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
@@ -151,7 +155,7 @@ public void setUp() throws Exception {
         }
         initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(metadata)
-            .routingTable(routingTable)
+            .routingTable(GlobalRoutingTable.builder().put(projectId, routingTable).build())
             .nodes(nb)
             .nodeIdsToCompatibilityVersions(compatibilityVersions)
             .build();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,36 +14,45 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
-    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
 
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
+    WOLFI(
+        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
         "-wolfi",
-        "apk"
+        "apk",
+        "Dockerfile"
+    ),
+    FIPS(
+        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        "-fips",
+        "apk",
+        "Dockerfile"
     ),
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
-    CLOUD_ESS(null, "-cloud-ess", "apk");
+    CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),;
 
     private final String image;
     private final String suffix;
     private final String packageManager;
+    private final String dockerfile;
 
     DockerBase(String image, String suffix) {
-        this(image, suffix, "apt-get");
+        this(image, suffix, "apt-get", "dockerfile");
     }
 
-    DockerBase(String image, String suffix, String packageManager) {
+    DockerBase(String image, String suffix, String packageManager, String dockerfile) {
         this.image = image;
         this.suffix = suffix;
         this.packageManager = packageManager;
+        this.dockerfile = dockerfile;
     }
 
     public String getImage() {
@@ -57,4 +66,8 @@ public String getSuffix() {
     public String getPackageManager() {
         return packageManager;
     }
+
+    public String getDockerfile() {
+        return dockerfile;
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/PatcherInfo.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.dependencies.patches;
+
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.function.Function;
+
+public final class PatcherInfo {
+    private final String jarEntryName;
+    private final byte[] classSha256;
+    private final Function<ClassWriter, ClassVisitor> visitorFactory;
+
+    private PatcherInfo(String jarEntryName, byte[] classSha256, Function<ClassWriter, ClassVisitor> visitorFactory) {
+        this.jarEntryName = jarEntryName;
+        this.classSha256 = classSha256;
+        this.visitorFactory = visitorFactory;
+    }
+
+    /**
+     * Creates a patcher info entry, linking a jar entry path name and its SHA256 digest to a patcher factory (a factory to create an ASM
+     * visitor)
+     *
+     * @param jarEntryName   the jar entry path, as a string
+     * @param classSha256    the SHA256 digest of the class bytes, as a HEX string
+     * @param visitorFactory the factory to create an ASM visitor from a ASM writer
+     */
+    public static PatcherInfo classPatcher(String jarEntryName, String classSha256, Function<ClassWriter, ClassVisitor> visitorFactory) {
+        return new PatcherInfo(jarEntryName, HexFormat.of().parseHex(classSha256), visitorFactory);
+    }
+
+    boolean matches(byte[] otherClassSha256) {
+        return Arrays.equals(this.classSha256, otherClassSha256);
+    }
+
+    public String jarEntryName() {
+        return jarEntryName;
+    }
+
+    public byte[] classSha256() {
+        return classSha256;
+    }
+
+    public ClassVisitor createVisitor(ClassWriter classWriter) {
+        return visitorFactory.apply(classWriter);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dependencies/patches/Utils.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.dependencies.patches;
+
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassWriter;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HexFormat;
+import java.util.Locale;
+import java.util.function.Function;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.stream.Collectors;
+
+import static org.objectweb.asm.ClassWriter.COMPUTE_FRAMES;
+import static org.objectweb.asm.ClassWriter.COMPUTE_MAXS;
+
+public class Utils {
+
+    private static final MessageDigest SHA_256;
+
+    static {
+        try {
+            SHA_256 = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private record MismatchInfo(String jarEntryName, String expectedClassSha256, String foundClassSha256) {
+        @Override
+        public String toString() {
+            return "[class='"
+                + jarEntryName
+                + '\''
+                + ", expected='"
+                + expectedClassSha256
+                + '\''
+                + ", found='"
+                + foundClassSha256
+                + '\''
+                + ']';
+        }
+    }
+
+    /**
+     * Patches the classes in the input JAR file, using the collection of patchers. Each patcher specifies a target class (its jar entry
+     * name) and the SHA256 digest on the class bytes.
+     * This digest is checked against the class bytes in the JAR, and if it does not match, an IllegalArgumentException is thrown.
+     * If the input file does not contain all the classes to patch specified in the patcher info collection, an IllegalArgumentException
+     * is also thrown.
+     * @param inputFile the JAR file to patch
+     * @param outputFile the output (patched) JAR file
+     * @param patchers list of patcher info (classes to patch (jar entry name + optional SHA256 digest) and ASM visitor to transform them)
+     */
+    public static void patchJar(File inputFile, File outputFile, Collection<PatcherInfo> patchers) {
+        var classPatchers = patchers.stream().collect(Collectors.toMap(PatcherInfo::jarEntryName, Function.identity()));
+        var mismatchedClasses = new ArrayList<MismatchInfo>();
+        try (JarFile jarFile = new JarFile(inputFile); JarOutputStream jos = new JarOutputStream(new FileOutputStream(outputFile))) {
+            Enumeration<JarEntry> entries = jarFile.entries();
+            while (entries.hasMoreElements()) {
+                JarEntry entry = entries.nextElement();
+                String entryName = entry.getName();
+                // Add the entry to the new JAR file
+                jos.putNextEntry(new JarEntry(entryName));
+
+                var classPatcher = classPatchers.remove(entryName);
+                if (classPatcher != null) {
+                    byte[] classToPatch = jarFile.getInputStream(entry).readAllBytes();
+                    var classSha256 = SHA_256.digest(classToPatch);
+
+                    if (classPatcher.matches(classSha256)) {
+                        ClassReader classReader = new ClassReader(classToPatch);
+                        ClassWriter classWriter = new ClassWriter(classReader, COMPUTE_MAXS | COMPUTE_FRAMES);
+                        classReader.accept(classPatcher.createVisitor(classWriter), 0);
+                        jos.write(classWriter.toByteArray());
+                    } else {
+                        mismatchedClasses.add(
+                            new MismatchInfo(
+                                classPatcher.jarEntryName(),
+                                HexFormat.of().formatHex(classPatcher.classSha256()),
+                                HexFormat.of().formatHex(classSha256)
+                            )
+                        );
+                    }
+                } else {
+                    // Read the entry's data and write it to the new JAR
+                    try (InputStream is = jarFile.getInputStream(entry)) {
+                        is.transferTo(jos);
+                    }
+                }
+                jos.closeEntry();
+            }
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        if (mismatchedClasses.isEmpty() == false) {
+            throw new IllegalArgumentException(
+                String.format(
+                    Locale.ROOT,
+                    """
+                        Error patching JAR [%s]: SHA256 digest mismatch (%s). This JAR was updated to a version that contains different \
+                        classes, for which this patcher was not designed. Please check if the patcher still \
+                        applies correctly, and update the SHA256 digest(s).""",
+                    inputFile.getName(),
+                    mismatchedClasses.stream().map(MismatchInfo::toString).collect(Collectors.joining())
+                )
+            );
+        }
+
+        if (classPatchers.isEmpty() == false) {
+            throw new IllegalArgumentException(
+                String.format(
+                    Locale.ROOT,
+                    "error patching [%s]: the jar does not contain [%s]",
+                    inputFile.getName(),
+                    String.join(", ", classPatchers.keySet())
+                )
+            );
+        }
+    }
+}