Merge branch 'main' into non-issue/esql-dense-vector-support-normalization

Author: carlosdelest

.buildkite/pull-requests.json

@@ -15,6 +15,19 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-performance-esbench-pr",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
     }
   ]
 }

build-tools-internal/build.gradle

@@ -220,13 +220,13 @@ gradlePlugin {
       id = 'elasticsearch.internal-yaml-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
-    transportVersionManagementPlugin {
-      id = 'elasticsearch.transport-version-management'
-      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
+    transportVersionReferencesPlugin {
+      id = 'elasticsearch.transport-version-references'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin'
     }
-    globalTransportVersionManagementPlugin {
-      id = 'elasticsearch.global-transport-version-management'
-      implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
+    transportVersionResourcesPlugin {
+      id = 'elasticsearch.transport-version-resources'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin'
     }
   }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy

@@ -92,8 +92,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
 
         file("myserver/build.gradle") << """
             apply plugin: 'java-library'
-            apply plugin: 'elasticsearch.transport-version-management'
-            apply plugin: 'elasticsearch.global-transport-version-management'
+            apply plugin: 'elasticsearch.transport-version-references'
+            apply plugin: 'elasticsearch.transport-version-resources'
         """
         definedTransportVersion("existing_91", "8012000")
         definedTransportVersion("existing_92", "8123000,8012001")
@@ -112,7 +112,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
 
         file("myplugin/build.gradle") << """
             apply plugin: 'java-library'
-            apply plugin: 'elasticsearch.transport-version-management'
+            apply plugin: 'elasticsearch.transport-version-references'
 
             dependencies {
                 implementation project(":myserver")

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -15,7 +15,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
-import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
+import org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -37,7 +37,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
-        project.getPluginManager().apply(TransportVersionManagementPlugin.class);
+        project.getPluginManager().apply(TransportVersionReferencesPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java

@@ -61,7 +61,7 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask
 
     @TaskAction
     public void checkTransportVersion() throws IOException {
-        var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
+        var results = new HashSet<TransportVersionReference>();
 
         for (var cpElement : getClassPath()) {
             Path file = cpElement.toPath();
@@ -74,8 +74,7 @@ public void checkTransportVersion() throws IOException {
         Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
     }
 
-    private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path basePath)
-        throws IOException {
+    private void addNamesFromClassesDirectory(Set<TransportVersionReference> results, Path basePath) throws IOException {
         Files.walkFileTree(basePath, new SimpleFileVisitor<>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
@@ -90,8 +89,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         });
     }
 
-    private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
-        throws IOException {
+    private void addNamesFromClass(Set<TransportVersionReference> results, InputStream classBytes, String classname) throws IOException {
         ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
             @Override
             public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
@@ -111,7 +109,7 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
                             if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
                                 && ldcInsnNode.cst instanceof String tvName
                                 && tvName.isEmpty() == false) {
-                                results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
+                                results.add(new TransportVersionReference(tvName, location));
                             } else {
                                 // The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
                                 throw new RuntimeException(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import java.util.ArrayList;
+import java.util.List;
+
+record TransportVersionDefinition(String name, List<TransportVersionId> ids) {
+    public static TransportVersionDefinition fromString(String filename, String contents) {
+        assert filename.endsWith(".csv");
+        String name = filename.substring(0, filename.length() - 4);
+        List<TransportVersionId> ids = new ArrayList<>();
+
+        if (contents.isEmpty() == false) {
+            for (String rawId : contents.split(",")) {
+                try {
+                    ids.add(TransportVersionId.fromString(rawId));
+                } catch (NumberFormatException e) {
+                    throw new IllegalStateException("Failed to parse id " + rawId + " in " + filename, e);
+                }
+            }
+        }
+
+        return new TransportVersionDefinition(name, ids);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable<TransportVersionId> {
+
+    static TransportVersionId fromString(String s) {
+        int complete = Integer.parseInt(s);
+        int patch = complete % 100;
+        int subsidiary = (complete / 100) % 10;
+        int server = (complete / 1000) % 1000;
+        int major = complete / 1000000;
+        return new TransportVersionId(complete, major, server, subsidiary, patch);
+    }
+
+    @Override
+    public int compareTo(TransportVersionId o) {
+        return Integer.compare(complete, o.complete);
+    }
+
+    @Override
+    public String toString() {
+        return Integer.toString(complete);
+    }
+
+    public int base() {
+        return (complete / 1000) * 1000;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+record TransportVersionLatest(String branch, String name, TransportVersionId id) {
+    public static TransportVersionLatest fromString(String filename, String contents) {
+        assert filename.endsWith(".csv");
+        String branch = filename.substring(0, filename.length() - 4);
+
+        String[] parts = contents.split(",");
+        if (parts.length != 2) {
+            throw new IllegalStateException("Invalid transport version latest file [" + filename + "]: " + contents);
+        }
+
+        return new TransportVersionLatest(branch, parts[0], TransportVersionId.fromString(parts[1]));
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReference.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.attributes.AttributeContainer;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE;
+
+record TransportVersionReference(String name, String location) {
+
+    private static final Attribute<Boolean> REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class);
+
+    static List<TransportVersionReference> listFromFile(Path file) throws IOException {
+        assert file.endsWith(".csv");
+        List<TransportVersionReference> results = new ArrayList<>();
+        for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) {
+            String[] parts = line.split(",", 2);
+            if (parts.length != 2) {
+                throw new IOException("Invalid transport version data file [" + file + "]: " + line);
+            }
+            results.add(new TransportVersionReference(parts[0], parts[1]));
+        }
+        return results;
+    }
+
+    static void addArtifactAttribute(AttributeContainer attributes) {
+        attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv");
+        attributes.attribute(REFERENCES_ATTRIBUTE, true);
+    }
+
+    @Override
+    public String toString() {
+        return name + "," + location;
+    }
+}

…rt/TransportVersionManagementPlugin.java …rt/TransportVersionReferencesPlugin.javabuild-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java renamed to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java renamed to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionReferencesPlugin.java

@@ -20,7 +20,7 @@
 import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getDefinitionsDirectory;
 import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getResourcesDirectory;
 
-public class TransportVersionManagementPlugin implements Plugin<Project> {
+public class TransportVersionReferencesPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
@@ -38,7 +38,7 @@ public void apply(Project project) {
         Configuration tvReferencesConfig = project.getConfigurations().create("transportVersionReferences", c -> {
             c.setCanBeConsumed(true);
             c.setCanBeResolved(false);
-            c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
+            c.attributes(TransportVersionReference::addArtifactAttribute);
         });
         project.getArtifacts().add(tvReferencesConfig.getName(), collectTask);