Merge branch 'main' into feature/esql-match-options

# Conflicts:
#	docs/reference/esql/functions/kibana/definition/match_operator.json
#	docs/reference/esql/functions/kibana/docs/match_operator.md
#	server/src/main/java/org/elasticsearch/TransportVersions.java
#	x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MapExpression.java
#	x-pack/plugin/esql/qa/testFixtures/src/main/resources/map-functions.csv-spec
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/map/LogWithBaseInMap.java
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
#	x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java
#	x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java

Author: carlosdelest

.buildkite/scripts/dra-workflow.sh

@@ -6,7 +6,7 @@ WORKFLOW="${DRA_WORKFLOW:-snapshot}"
 BRANCH="${BUILDKITE_BRANCH:-}"
 
 # Don't publish main branch to staging
-if [[ ("$BRANCH" == "main" || "$BRANCH" == *.x) && "$WORKFLOW" == "staging" ]]; then
+if [[ "$BRANCH" == *.x && "$WORKFLOW" == "staging" ]]; then
   exit 0
 fi
 
@@ -22,13 +22,19 @@ if [[ "$BRANCH" == "main" ]]; then
 fi
 
 ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g")
+BASE_VERSION="$ES_VERSION"
 echo "ES_VERSION=$ES_VERSION"
 
 VERSION_SUFFIX=""
 if [[ "$WORKFLOW" == "snapshot" ]]; then
   VERSION_SUFFIX="-SNAPSHOT"
 fi
 
+if [[ -n "${VERSION_QUALIFIER:-}" ]]; then
+  ES_VERSION="${ES_VERSION}-${VERSION_QUALIFIER}"
+  echo "Version qualifier specified. ES_VERSION=${ES_VERSION}."
+fi
+
 BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
 echo "BEATS_BUILD_ID=$BEATS_BUILD_ID"
 
@@ -37,6 +43,7 @@ echo "ML_CPP_BUILD_ID=$ML_CPP_BUILD_ID"
 
 LICENSE_KEY_ARG=""
 BUILD_SNAPSHOT_ARG=""
+VERSION_QUALIFIER_ARG=""
 
 if [[ "$WORKFLOW" == "staging" ]]; then
   LICENSE_KEY=$(mktemp -d)/license.key
@@ -47,6 +54,10 @@ if [[ "$WORKFLOW" == "staging" ]]; then
   BUILD_SNAPSHOT_ARG="-Dbuild.snapshot=false"
 fi
 
+if [[ -n "${VERSION_QUALIFIER:-}" ]]; then
+  VERSION_QUALIFIER_ARG="-Dbuild.version_qualifier=$VERSION_QUALIFIER"
+fi
+
 echo --- Building release artifacts
 
 .ci/scripts/run-gradle.sh -Ddra.artifacts=true \
@@ -56,12 +67,17 @@ echo --- Building release artifacts
   -Dcsv="$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv" \
   $LICENSE_KEY_ARG \
   $BUILD_SNAPSHOT_ARG \
+  $VERSION_QUALIFIER_ARG \
   buildReleaseArtifacts \
   exportCompressedDockerImages \
   :distribution:generateDependenciesReport
 
 PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script
+if [[ -z "${VERSION_QUALIFIER:-}" ]]; then
 x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
+else
+x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="-$VERSION_QUALIFIER"
+fi
 
 # we regenerate this file as part of the release manager invocation
 rm "build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512"
@@ -88,7 +104,8 @@ docker run --rm \
   --branch "$RM_BRANCH" \
   --commit "$BUILDKITE_COMMIT" \
   --workflow "$WORKFLOW" \
-  --version "$ES_VERSION" \
+  --qualifier "${VERSION_QUALIFIER:-}" \
+  --version "$BASE_VERSION" \
   --artifact-set main \
   --dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
   --dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"

.buildkite/scripts/dra-workflow.trigger.sh

@@ -8,7 +8,7 @@ source .buildkite/scripts/branches.sh
 
 for BRANCH in "${BRANCHES[@]}"; do
   if [[ "$BRANCH" == "main" ]]; then
-    continue
+    export VERSION_QUALIFIER="alpha1"
   fi
 
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
@@ -24,5 +24,6 @@ for BRANCH in "${BRANCHES[@]}"; do
       commit: "$LAST_GOOD_COMMIT"
       env:
         DRA_WORKFLOW: staging
+        VERSION_QUALIFIER: ${VERSION_QUALIFIER:-}
 EOF
 done

.buildkite/scripts/get-latest-test-mutes.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false"  || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
   exit 0
 fi
 

benchmarks/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.internal.test.TestUtil
+import org.elasticsearch.gradle.OS
 
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
@@ -77,7 +78,7 @@ tasks.register("copyPainless", Copy) {
 }
 
 tasks.named("run").configure {
-  executable = "${buildParams.runtimeJavaHome.get()}/bin/java"
+  executable = "${buildParams.runtimeJavaHome.get()}/bin/java" + (OS.current() == OS.WINDOWS ? '.exe' : '')
   args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index"
   dependsOn "copyExpression", "copyPainless", configurations.nativeLib
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayReadLongBenchmark.java

@@ -47,7 +47,7 @@ public class BytesArrayReadLongBenchmark {
     @Setup
     public void initResults() throws IOException {
         final BytesStreamOutput tmp = new BytesStreamOutput();
-        final long bytes = new ByteSizeValue(dataMb, ByteSizeUnit.MB).getBytes();
+        final long bytes = ByteSizeValue.of(dataMb, ByteSizeUnit.MB).getBytes();
         for (int i = 0; i < bytes / 8; i++) {
             tmp.writeLong(i);
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/PagedBytesReferenceReadLongBenchmark.java

@@ -47,7 +47,7 @@ public class PagedBytesReferenceReadLongBenchmark {
     @Setup
     public void initResults() throws IOException {
         final BytesStreamOutput tmp = new BytesStreamOutput();
-        final long bytes = new ByteSizeValue(dataMb, ByteSizeUnit.MB).getBytes();
+        final long bytes = ByteSizeValue.of(dataMb, ByteSizeUnit.MB).getBytes();
         for (int i = 0; i < bytes / 8; i++) {
             tmp.writeLong(i);
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -71,12 +72,11 @@ public class EvalBenchmark {
         BigArrays.NON_RECYCLING_INSTANCE
     );
 
+    private static final FoldContext FOLD_CONTEXT = FoldContext.small();
+
     private static final int BLOCK_LENGTH = 8 * 1024;
 
-    static final DriverContext driverContext = new DriverContext(
-        BigArrays.NON_RECYCLING_INSTANCE,
-        BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
-    );
+    static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);
 
     static {
         // Smoke test all the expected values and force loading subclasses more like prod
@@ -114,18 +114,20 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
         return switch (operation) {
             case "abs" -> {
                 FieldAttribute longField = longField();
-                yield EvalMapper.toEvaluator(new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
             case "add" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
             }
             case "add_double" -> {
                 FieldAttribute doubleField = doubleField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)),
                     layout(doubleField)
                 ).get(driverContext);
@@ -140,7 +142,8 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
                     rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
                 }
-                yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2))
+                    .get(driverContext);
             }
             case "date_trunc" -> {
                 FieldAttribute timestamp = new FieldAttribute(
@@ -149,35 +152,37 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new EsField("timestamp", DataType.DATETIME, Map.of(), true)
                 );
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
                     layout(timestamp)
                 ).get(driverContext);
             }
             case "equal_to_const" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
             }
             case "long_equal_to_long" -> {
                 FieldAttribute lhs = longField();
                 FieldAttribute rhs = longField();
-                yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
             }
             case "long_equal_to_int" -> {
                 FieldAttribute lhs = longField();
                 FieldAttribute rhs = intField();
-                yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
             }
             case "mv_min", "mv_min_ascending" -> {
                 FieldAttribute longField = longField();
-                yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
             case "rlike" -> {
                 FieldAttribute keywordField = keywordField();
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
-                yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
             default -> throw new UnsupportedOperationException();
         };

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:eef54b3a414aa53b98f0f8df2633aed83c3ba6230722769282925442968f0364",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:dd66beec64a7f9b19c6c35a1195153b2b630a55e16ec71949ed5187c5947eea1",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -53,7 +53,7 @@ public void apply(Project project) {
     }
 
     // List of root tasks, by name, whose compileJava task should not use the module path. These are test related sources.
-    static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common");
+    static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common", ":x-pack:plugin:esql:compute:test");
 
     void configureCompileModulePath(Project project) {
         // first disable Gradle's builtin module path inference

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -26,7 +26,7 @@ public interface TestClustersAware extends Task {
     Collection<ElasticsearchCluster> getClusters();
 
     @ServiceReference(REGISTRY_SERVICE_NAME)
-    Property<TestClustersRegistry> getRegistery();
+    Property<TestClustersRegistry> getRegistry();
 
     @ServiceReference(TEST_CLUSTER_TASKS_SERVICE)
     Property<TestClustersPlugin.TaskEventsService> getTasksService();
@@ -47,6 +47,14 @@ default void useCluster(ElasticsearchCluster cluster) {
         getClusters().add(cluster);
     }
 
+    default Provider<TestClusterInfo> getClusterInfo(String clusterName) {
+        return getProject().getProviders().of(TestClusterValueSource.class, source -> {
+            source.getParameters().getService().set(getRegistry());
+            source.getParameters().getClusterName().set(clusterName);
+            source.getParameters().getPath().set(getProject().getIsolated().getPath());
+        });
+    }
+
     default void useCluster(Provider<ElasticsearchCluster> cluster) {
         useCluster(cluster.get());
     }