Merge remote-tracking branch 'upstream/main' into dls-expose-existing-stats

* upstream/main: (36 commits)
  ESQL: Fix async operator warnings not always sent when blocking (elastic#132744)
  Method not needed anymore (elastic#132912)
  [Test] Excercise shutdown more reliably in snapshot stress IT (elastic#132909)
  Update Gradle shadow plugin to 9.0.1 (elastic#132637)
  Mute org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT test {p0=search/410_named_queries/named_queries_with_score} elastic#132906
  Update docker.elastic.co/wolfi/chainguard-base-fips:latest Docker digest to fa6cb69 (elastic#132735)
  Remove unnecessary calls to fold()  (elastic#131870)
  Use consistent terminology for transport version resources/references (elastic#132882)
  Mute org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT test {p0=search.vectors/40_knn_search_cosine/kNN search only regular query} elastic#132890
  Finalize release notes for v9.1.2 release (elastic#132745)
  Finalize release notes for v9.0.5 release (elastic#132718)
  Move inner records out of TransportVersionUtils (elastic#132872)
  Add support for Lookup Join on Multiple Fields (elastic#131559)
  Bootstrap PR-based benchmarks (elastic#132717)
  Refactor MetadataIndexTemplateService to use template maps instead of project metadata (elastic#132662)
  [Gradle] Update nebula ospackage plugin to 12.1.0 (elastic#132640)
  Mute org.elasticsearch.xpack.esql.CsvTests test {csv-spec:ip.CdirMatchEqualsInsOrs} elastic#132860
  Mute org.elasticsearch.xpack.esql.CsvTests test {csv-spec:floats.InMultivalue} elastic#132859
  Revert "Reuse prod code and reduce EsqlSession public surface" (elastic#132843)
  Mute org.elasticsearch.xpack.esql.CsvTests test {csv-spec:string.LengthOfText} elastic#132857
  ...

Author: szybia

.buildkite/pull-requests.json

@@ -15,6 +15,19 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-performance-esbench-pr",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
     }
   ]
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.benchmark.vector;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.BQVectorUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 4, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 5, time = 1)
+// engage some noise reduction
+@Fork(value = 1)
+public class PackAsBinaryBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    @Param({ "384", "782", "1024" })
+    int dims;
+
+    int length;
+
+    int numVectors = 1000;
+
+    int[][] qVectors;
+    byte[] packed;
+
+    @Setup
+    public void setup() throws IOException {
+        Random random = new Random(123);
+
+        this.length = BQVectorUtils.discretize(dims, 64) / 8;
+        this.packed = new byte[length];
+
+        qVectors = new int[numVectors][dims];
+        for (int[] qVector : qVectors) {
+            for (int i = 0; i < dims; i++) {
+                qVector[i] = random.nextInt(2);
+            }
+        }
+    }
+
+    @Benchmark
+    public void packAsBinary(Blackhole bh) {
+        for (int i = 0; i < numVectors; i++) {
+            BQVectorUtils.packAsBinary(qVectors[i], packed);
+            bh.consume(packed);
+        }
+    }
+
+    @Benchmark
+    public void packAsBinaryLegacy(Blackhole bh) {
+        for (int i = 0; i < numVectors; i++) {
+            BQVectorUtils.packAsBinaryLegacy(qVectors[i], packed);
+            bh.consume(packed);
+        }
+    }
+}

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -12,7 +12,6 @@
 import groovy.util.Node;
 import nmcp.NmcpPlugin;
 
-import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
 
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
@@ -173,8 +172,7 @@ private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<Ma
     }
 
     private static void configureWithShadowPlugin(Project project, MavenPublication publication) {
-        var shadow = project.getExtensions().getByType(ShadowExtension.class);
-        shadow.component(publication);
+        publication.from(project.getComponents().getByName("shadow"));
         publication.artifact(project.getTasks().named("javadocJar"));
         publication.artifact(project.getTasks().named("sourcesJar"));
     }

build-tools-internal/build.gradle

@@ -220,13 +220,13 @@ gradlePlugin {
       id = 'elasticsearch.internal-yaml-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
-    transportVersionManagementPlugin {
-      id = 'elasticsearch.transport-version-management'
-      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
+    transportVersionReferencesPlugin {
+      id = 'elasticsearch.transport-version-references'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin'
     }
-    globalTransportVersionManagementPlugin {
-      id = 'elasticsearch.global-transport-version-management'
-      implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
+    transportVersionResourcesPlugin {
+      id = 'elasticsearch.transport-version-resources'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin'
     }
   }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy

@@ -92,8 +92,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
 
         file("myserver/build.gradle") << """
             apply plugin: 'java-library'
-            apply plugin: 'elasticsearch.transport-version-management'
-            apply plugin: 'elasticsearch.global-transport-version-management'
+            apply plugin: 'elasticsearch.transport-version-references'
+            apply plugin: 'elasticsearch.transport-version-resources'
         """
         definedTransportVersion("existing_91", "8012000")
         definedTransportVersion("existing_92", "8123000,8012001")
@@ -112,7 +112,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
 
         file("myplugin/build.gradle") << """
             apply plugin: 'java-library'
-            apply plugin: 'elasticsearch.transport-version-management'
+            apply plugin: 'elasticsearch.transport-version-references'
 
             dependencies {
                 implementation project(":myserver")

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -15,7 +15,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
-import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
+import org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -37,7 +37,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
-        project.getPluginManager().apply(TransportVersionManagementPlugin.class);
+        project.getPluginManager().apply(TransportVersionReferencesPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java

@@ -9,10 +9,9 @@
 
 package org.elasticsearch.gradle.internal.shadow;
 
-import com.github.jengelman.gradle.plugins.shadow.ShadowStats;
 import com.github.jengelman.gradle.plugins.shadow.relocation.RelocateClassContext;
 import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator;
-import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer;
+import com.github.jengelman.gradle.plugins.shadow.transformers.ResourceTransformer;
 import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext;
 
 import org.apache.commons.io.IOUtils;
@@ -26,7 +25,7 @@
 import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.List;
+import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -35,7 +34,7 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
-public class XmlClassRelocationTransformer implements Transformer {
+public class XmlClassRelocationTransformer implements ResourceTransformer {
 
     boolean hasTransformedResource = false;
 
@@ -55,7 +54,7 @@ public boolean canTransformResource(FileTreeElement element) {
     @Override
     public void transform(TransformerContext context) {
         try {
-            BufferedInputStream bis = new BufferedInputStream(context.getIs());
+            BufferedInputStream bis = new BufferedInputStream(context.getInputStream());
             DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
             DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
             doc = dBuilder.parse(bis);
@@ -66,17 +65,16 @@ public void transform(TransformerContext context) {
                 this.doc = null;
             }
         } catch (Exception e) {
-            throw new RuntimeException("Error parsing xml file in " + context.getIs(), e);
+            throw new RuntimeException("Error parsing xml file in " + context.getInputStream(), e);
         }
     }
 
     private static String getRelocatedClass(String className, TransformerContext context) {
-        List<Relocator> relocators = context.getRelocators();
-        ShadowStats stats = context.getStats();
+        Set<Relocator> relocators = context.getRelocators();
         if (className != null && className.length() > 0 && relocators != null) {
             for (Relocator relocator : relocators) {
                 if (relocator.canRelocateClass(className)) {
-                    RelocateClassContext relocateClassContext = new RelocateClassContext(className, stats);
+                    RelocateClassContext relocateClassContext = new RelocateClassContext(className);
                     return relocator.relocateClass(relocateClassContext);
                 }
             }
@@ -111,8 +109,6 @@ public boolean hasTransformedResource() {
     @Override
     public void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) {
         ZipEntry entry = new ZipEntry(resource);
-        entry.setTime(TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.getTime()));
-
         try {
             // Write the content back to the XML file
             TransformerFactory transformerFactory = TransformerFactory.newInstance();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java

@@ -61,7 +61,7 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask
 
     @TaskAction
     public void checkTransportVersion() throws IOException {
-        var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
+        var results = new HashSet<TransportVersionReference>();
 
         for (var cpElement : getClassPath()) {
             Path file = cpElement.toPath();
@@ -74,8 +74,7 @@ public void checkTransportVersion() throws IOException {
         Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
     }
 
-    private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path basePath)
-        throws IOException {
+    private void addNamesFromClassesDirectory(Set<TransportVersionReference> results, Path basePath) throws IOException {
         Files.walkFileTree(basePath, new SimpleFileVisitor<>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
@@ -90,8 +89,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         });
     }
 
-    private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
-        throws IOException {
+    private void addNamesFromClass(Set<TransportVersionReference> results, InputStream classBytes, String classname) throws IOException {
         ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
             @Override
             public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
@@ -111,7 +109,7 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
                             if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
                                 && ldcInsnNode.cst instanceof String tvName
                                 && tvName.isEmpty() == false) {
-                                results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
+                                results.add(new TransportVersionReference(tvName, location));
                             } else {
                                 // The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
                                 throw new RuntimeException(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import java.util.ArrayList;
+import java.util.List;
+
+record TransportVersionDefinition(String name, List<TransportVersionId> ids) {
+    public static TransportVersionDefinition fromString(String filename, String contents) {
+        assert filename.endsWith(".csv");
+        String name = filename.substring(0, filename.length() - 4);
+        List<TransportVersionId> ids = new ArrayList<>();
+
+        if (contents.isEmpty() == false) {
+            for (String rawId : contents.split(",")) {
+                try {
+                    ids.add(TransportVersionId.fromString(rawId));
+                } catch (NumberFormatException e) {
+                    throw new IllegalStateException("Failed to parse id " + rawId + " in " + filename, e);
+                }
+            }
+        }
+
+        return new TransportVersionDefinition(name, ids);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable<TransportVersionId> {
+
+    static TransportVersionId fromString(String s) {
+        int complete = Integer.parseInt(s);
+        int patch = complete % 100;
+        int subsidiary = (complete / 100) % 10;
+        int server = (complete / 1000) % 1000;
+        int major = complete / 1000000;
+        return new TransportVersionId(complete, major, server, subsidiary, patch);
+    }
+
+    @Override
+    public int compareTo(TransportVersionId o) {
+        return Integer.compare(complete, o.complete);
+    }
+
+    @Override
+    public String toString() {
+        return Integer.toString(complete);
+    }
+
+    public int base() {
+        return (complete / 1000) * 1000;
+    }
+}