Merge remote-tracking branch 'origin/main' into non-issue/esql-vector-similarity-null-management

# Conflicts:
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

Author: carlosdelest

.buildkite/pull-requests.json

@@ -15,6 +15,19 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-performance-esbench-pr",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
     }
   ]
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -12,7 +12,6 @@
 import groovy.util.Node;
 import nmcp.NmcpPlugin;
 
-import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
 
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
@@ -173,8 +172,7 @@ private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<Ma
     }
 
     private static void configureWithShadowPlugin(Project project, MavenPublication publication) {
-        var shadow = project.getExtensions().getByType(ShadowExtension.class);
-        shadow.component(publication);
+        publication.from(project.getComponents().getByName("shadow"));
         publication.artifact(project.getTasks().named("javadocJar"));
         publication.artifact(project.getTasks().named("sourcesJar"));
     }

build-tools-internal/build.gradle

@@ -220,13 +220,13 @@ gradlePlugin {
       id = 'elasticsearch.internal-yaml-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
-    transportVersionManagementPlugin {
-      id = 'elasticsearch.transport-version-management'
-      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
+    transportVersionReferencesPlugin {
+      id = 'elasticsearch.transport-version-references'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin'
     }
-    globalTransportVersionManagementPlugin {
-      id = 'elasticsearch.global-transport-version-management'
-      implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
+    transportVersionResourcesPlugin {
+      id = 'elasticsearch.transport-version-resources'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin'
     }
   }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy

@@ -92,8 +92,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
 
         file("myserver/build.gradle") << """
             apply plugin: 'java-library'
-            apply plugin: 'elasticsearch.transport-version-management'
-            apply plugin: 'elasticsearch.global-transport-version-management'
+            apply plugin: 'elasticsearch.transport-version-references'
+            apply plugin: 'elasticsearch.transport-version-resources'
         """
         definedTransportVersion("existing_91", "8012000")
         definedTransportVersion("existing_92", "8123000,8012001")
@@ -112,7 +112,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
 
         file("myplugin/build.gradle") << """
             apply plugin: 'java-library'
-            apply plugin: 'elasticsearch.transport-version-management'
+            apply plugin: 'elasticsearch.transport-version-references'
 
             dependencies {
                 implementation project(":myserver")

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -15,7 +15,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
-import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
+import org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -37,7 +37,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
-        project.getPluginManager().apply(TransportVersionManagementPlugin.class);
+        project.getPluginManager().apply(TransportVersionReferencesPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java

@@ -9,10 +9,9 @@
 
 package org.elasticsearch.gradle.internal.shadow;
 
-import com.github.jengelman.gradle.plugins.shadow.ShadowStats;
 import com.github.jengelman.gradle.plugins.shadow.relocation.RelocateClassContext;
 import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator;
-import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer;
+import com.github.jengelman.gradle.plugins.shadow.transformers.ResourceTransformer;
 import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext;
 
 import org.apache.commons.io.IOUtils;
@@ -26,7 +25,7 @@
 import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.List;
+import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -35,7 +34,7 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
-public class XmlClassRelocationTransformer implements Transformer {
+public class XmlClassRelocationTransformer implements ResourceTransformer {
 
     boolean hasTransformedResource = false;
 
@@ -55,7 +54,7 @@ public boolean canTransformResource(FileTreeElement element) {
     @Override
     public void transform(TransformerContext context) {
         try {
-            BufferedInputStream bis = new BufferedInputStream(context.getIs());
+            BufferedInputStream bis = new BufferedInputStream(context.getInputStream());
             DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
             DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
             doc = dBuilder.parse(bis);
@@ -66,17 +65,16 @@ public void transform(TransformerContext context) {
                 this.doc = null;
             }
         } catch (Exception e) {
-            throw new RuntimeException("Error parsing xml file in " + context.getIs(), e);
+            throw new RuntimeException("Error parsing xml file in " + context.getInputStream(), e);
         }
     }
 
     private static String getRelocatedClass(String className, TransformerContext context) {
-        List<Relocator> relocators = context.getRelocators();
-        ShadowStats stats = context.getStats();
+        Set<Relocator> relocators = context.getRelocators();
         if (className != null && className.length() > 0 && relocators != null) {
             for (Relocator relocator : relocators) {
                 if (relocator.canRelocateClass(className)) {
-                    RelocateClassContext relocateClassContext = new RelocateClassContext(className, stats);
+                    RelocateClassContext relocateClassContext = new RelocateClassContext(className);
                     return relocator.relocateClass(relocateClassContext);
                 }
             }
@@ -111,8 +109,6 @@ public boolean hasTransformedResource() {
     @Override
     public void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) {
         ZipEntry entry = new ZipEntry(resource);
-        entry.setTime(TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.getTime()));
-
         try {
             // Write the content back to the XML file
             TransformerFactory transformerFactory = TransformerFactory.newInstance();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java

@@ -61,7 +61,7 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask
 
     @TaskAction
     public void checkTransportVersion() throws IOException {
-        var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
+        var results = new HashSet<TransportVersionReference>();
 
         for (var cpElement : getClassPath()) {
             Path file = cpElement.toPath();
@@ -74,8 +74,7 @@ public void checkTransportVersion() throws IOException {
         Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
     }
 
-    private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path basePath)
-        throws IOException {
+    private void addNamesFromClassesDirectory(Set<TransportVersionReference> results, Path basePath) throws IOException {
         Files.walkFileTree(basePath, new SimpleFileVisitor<>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
@@ -90,8 +89,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         });
     }
 
-    private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
-        throws IOException {
+    private void addNamesFromClass(Set<TransportVersionReference> results, InputStream classBytes, String classname) throws IOException {
         ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
             @Override
             public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
@@ -111,7 +109,7 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
                             if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
                                 && ldcInsnNode.cst instanceof String tvName
                                 && tvName.isEmpty() == false) {
-                                results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
+                                results.add(new TransportVersionReference(tvName, location));
                             } else {
                                 // The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
                                 throw new RuntimeException(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import java.util.ArrayList;
+import java.util.List;
+
+record TransportVersionDefinition(String name, List<TransportVersionId> ids) {
+    public static TransportVersionDefinition fromString(String filename, String contents) {
+        assert filename.endsWith(".csv");
+        String name = filename.substring(0, filename.length() - 4);
+        List<TransportVersionId> ids = new ArrayList<>();
+
+        if (contents.isEmpty() == false) {
+            for (String rawId : contents.split(",")) {
+                try {
+                    ids.add(TransportVersionId.fromString(rawId));
+                } catch (NumberFormatException e) {
+                    throw new IllegalStateException("Failed to parse id " + rawId + " in " + filename, e);
+                }
+            }
+        }
+
+        return new TransportVersionDefinition(name, ids);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable<TransportVersionId> {
+
+    static TransportVersionId fromString(String s) {
+        int complete = Integer.parseInt(s);
+        int patch = complete % 100;
+        int subsidiary = (complete / 100) % 10;
+        int server = (complete / 1000) % 1000;
+        int major = complete / 1000000;
+        return new TransportVersionId(complete, major, server, subsidiary, patch);
+    }
+
+    @Override
+    public int compareTo(TransportVersionId o) {
+        return Integer.compare(complete, o.complete);
+    }
+
+    @Override
+    public String toString() {
+        return Integer.toString(complete);
+    }
+
+    public int base() {
+        return (complete / 1000) * 1000;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+record TransportVersionLatest(String branch, String name, TransportVersionId id) {
+    public static TransportVersionLatest fromString(String filename, String contents) {
+        assert filename.endsWith(".csv");
+        String branch = filename.substring(0, filename.length() - 4);
+
+        String[] parts = contents.split(",");
+        if (parts.length != 2) {
+            throw new IllegalStateException("Invalid transport version latest file [" + filename + "]: " + contents);
+        }
+
+        return new TransportVersionLatest(branch, parts[0], TransportVersionId.fromString(parts[1]));
+    }
+}