Merge branch 'main' into s3-failIfAlreadyExists

Author: kvanerum

.buildkite/pipelines/pull-request-performance-benchmark.yml

@@ -0,0 +1,3 @@
+steps:
+  - label: ":pipeline: Generate steps"
+    command: bash .buildkite/scripts/generate-pr-performance-benchmark.sh | buildkite-agent pipeline upload

.buildkite/pull-requests.json

@@ -15,6 +15,20 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-pull-request-performance-benchmark",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "target_branch": "main",
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\S+)( please)?$"
     }
   ]
 }

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
+env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
+merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
+
+buildkite-agent meta-data set pr_comment:custom-body:body \
+  "This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+buildkite-agent meta-data set pr_comment:custom-baseline:head \
+  "* Baseline: ${merge_base} (env ID ${env_id_baseline})"
+buildkite-agent meta-data set pr_comment:custom-contender:head \
+  "* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
+
+cat << _EOF_
+steps:
+  - label: Trigger baseline benchmark
+    trigger: elasticsearch-performance-esbench-pr
+    build:
+      message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
+      branch: master
+      env:
+        CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
+        ENV_ID: ${env_id_baseline}
+        REVISION: ${merge_base}
+  - label: Trigger contender benchmark
+    trigger: elasticsearch-performance-esbench-pr
+    build:
+      message: Contender benchmark for PR${GITHUB_PR_NUMBER}
+      branch: master
+      env:
+        CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
+        ENV_ID: ${env_id_contender}
+        ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
+        REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+  - wait: ~
+  - label: Modify PR comment
+    command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
+_EOF_

.gitignore

@@ -1,4 +1,7 @@
 
+# claude
+.claude
+
 # intellij files
 .idea/
 *.iml
@@ -69,7 +72,7 @@ testfixtures_shared/
 # Generated
 checkstyle_ide.xml
 x-pack/plugin/esql/src/main/generated-src/generated/
-server/src/main/resources/transport/defined/manifest.txt
+server/src/main/resources/transport/definitions/manifest.txt
 
 # JEnv
 .java-version

BUILDING.md

@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
-> [!Tip]  
+> [!Tip]
 > A manual confirmation of the Gradle generated checksums is currently not mandatory.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
@@ -186,6 +186,25 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
+### How do I test against java early access (ea) versions?
+
+Currently only openjdk EA builds by oracle are supported.
+To test against an early access version java version you can pass the major
+java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+
+```
+./gradlew clean test -Druntime.java=26-ea
+```
+
+This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+
+To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
+
+```
+./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+```
+
 #### How to use a Maven based third party dependency via `mavenlocal`?
 
 1. Clone the third party repository locally
@@ -229,7 +248,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
-> [!Note]  
+> [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 
@@ -261,7 +280,7 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
-> [!Note]  
+> [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository

benchmarks/build.gradle

@@ -47,6 +47,7 @@ dependencies {
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
+  api(project(':x-pack:plugin:mapper-exponential-histogram'))
   implementation project(path: ':libs:native')
   implementation project(path: ':libs:simdvec')
   implementation project(path: ':libs:exponential-histogram')

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -92,7 +92,8 @@ public void setup() {
         var fields = 10_000;
         var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
         for (int i = 0; i < fields; i++) {
-            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
+            // We're creating a standard index, so none of these fields should be marked as dimensions.
+            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE));
         }
 
         var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -142,11 +142,26 @@ static void selfTest() {
     private static List<ValuesSourceReaderOperator.FieldInfo> fields(String name) {
         return switch (name) {
             case "3_stored_keywords" -> List.of(
-                new ValuesSourceReaderOperator.FieldInfo("keyword_1", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_1")),
-                new ValuesSourceReaderOperator.FieldInfo("keyword_2", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_2")),
-                new ValuesSourceReaderOperator.FieldInfo("keyword_3", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_3"))
+                new ValuesSourceReaderOperator.FieldInfo(
+                    "keyword_1",
+                    ElementType.BYTES_REF,
+                    false,
+                    shardIdx -> blockLoader("stored_keyword_1")
+                ),
+                new ValuesSourceReaderOperator.FieldInfo(
+                    "keyword_2",
+                    ElementType.BYTES_REF,
+                    false,
+                    shardIdx -> blockLoader("stored_keyword_2")
+                ),
+                new ValuesSourceReaderOperator.FieldInfo(
+                    "keyword_3",
+                    ElementType.BYTES_REF,
+                    false,
+                    shardIdx -> blockLoader("stored_keyword_3")
+                )
             );
-            default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), shardIdx -> blockLoader(name)));
+            default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), false, shardIdx -> blockLoader(name)));
         };
     }
 

benchmarks/src/main/java/org/elasticsearch/benchmark/common/network/IpAddressesBenchmarks.java

@@ -0,0 +1,142 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.common.network;
+
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.xcontent.Text;
+import org.elasticsearch.xcontent.XContentString;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 2)
+@Measurement(iterations = 3)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+@Fork(1)
+public class IpAddressesBenchmarks {
+
+    @Param("1000")
+    private int size;
+    private String[] ipV6Addresses;
+    private String[] ipV4Addresses;
+    private XContentString[] ipV6AddressesBytes;
+    private XContentString[] ipV4AddressesBytes;
+
+    @Setup
+    public void setup() throws UnknownHostException {
+        Random random = new Random();
+        ipV6Addresses = new String[size];
+        ipV4Addresses = new String[size];
+        ipV6AddressesBytes = new XContentString[size];
+        ipV4AddressesBytes = new XContentString[size];
+        byte[] ipv6Bytes = new byte[16];
+        byte[] ipv4Bytes = new byte[4];
+        for (int i = 0; i < size; i++) {
+            random.nextBytes(ipv6Bytes);
+            random.nextBytes(ipv4Bytes);
+            String ipv6String = InetAddresses.toAddrString(InetAddress.getByAddress(ipv6Bytes));
+            String ipv4String = InetAddresses.toAddrString(InetAddress.getByAddress(ipv4Bytes));
+            ipV6Addresses[i] = ipv6String;
+            ipV4Addresses[i] = ipv4String;
+            ipV6AddressesBytes[i] = new Text(ipv6String);
+            ipV4AddressesBytes[i] = new Text(ipv4String);
+        }
+    }
+
+    @Benchmark
+    public boolean isInetAddressIpv6() {
+        boolean b = true;
+        for (int i = 0; i < size; i++) {
+            b ^= InetAddresses.isInetAddress(ipV6Addresses[i]);
+        }
+        return b;
+    }
+
+    @Benchmark
+    public boolean isInetAddressIpv4() {
+        boolean b = true;
+        for (int i = 0; i < size; i++) {
+            b ^= InetAddresses.isInetAddress(ipV4Addresses[i]);
+        }
+        return b;
+    }
+
+    @Benchmark
+    public void getIpOrHostIpv6(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.getIpOrHost(ipV6Addresses[i]));
+        }
+    }
+
+    @Benchmark
+    public void getIpOrHostIpv4(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.forString(ipV4Addresses[i]));
+        }
+    }
+
+    @Benchmark
+    public void forStringIpv6String(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.forString(ipV6Addresses[i]));
+        }
+    }
+
+    @Benchmark
+    public void forStringIpv4String(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.forString(ipV4Addresses[i]));
+        }
+    }
+
+    @Benchmark
+    public void forStringIpv6Bytes(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.forString(ipV6AddressesBytes[i].bytes()));
+        }
+    }
+
+    @Benchmark
+    public void forStringIpv4Bytes(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.forString(ipV4AddressesBytes[i].bytes()));
+        }
+    }
+
+    @Benchmark
+    public void encodeAsIpv6WithIpv6(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.encodeAsIpv6(ipV6AddressesBytes[i]));
+        }
+    }
+
+    @Benchmark
+    public void encodeAsIpv6WithIpv4(Blackhole blackhole) {
+        for (int i = 0; i < size; i++) {
+            blackhole.consume(InetAddresses.encodeAsIpv6(ipV4AddressesBytes[i]));
+        }
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -212,7 +212,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 FieldAttribute timestamp = new FieldAttribute(
                     Source.EMPTY,
                     "timestamp",
-                    new EsField("timestamp", DataType.DATETIME, Map.of(), true)
+                    new EsField("timestamp", DataType.DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
                 );
                 yield EvalMapper.toEvaluator(
                     FOLD_CONTEXT,
@@ -321,19 +321,35 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
     }
 
     private static FieldAttribute longField() {
-        return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataType.LONG, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "long",
+            new EsField("long", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static FieldAttribute doubleField() {
-        return new FieldAttribute(Source.EMPTY, "double", new EsField("double", DataType.DOUBLE, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "double",
+            new EsField("double", DataType.DOUBLE, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static FieldAttribute intField() {
-        return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataType.INTEGER, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "int",
+            new EsField("int", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static FieldAttribute keywordField() {
-        return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "keyword",
+            new EsField("keyword", DataType.KEYWORD, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static Configuration configuration() {