Merge branch 'main' into google-vertexai-chatcompletion

Author: lhoet-google

.buildkite/pipelines/periodic-packaging.template.yml

@@ -40,6 +40,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic-packaging.yml

@@ -393,6 +393,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic-platform-support.yml

@@ -38,6 +38,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
             GRADLE_TASK:
               - checkPart1
               - checkPart2

.buildkite/pipelines/pull-request/packaging-tests-windows.yml

@@ -11,6 +11,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
             PACKAGING_TASK:
               - default-windows-archive
         agents:

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -44,13 +45,15 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
+import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.session.Configuration;
@@ -128,6 +131,10 @@ static void selfTest() {
             "long_equal_to_int",
             "mv_min",
             "mv_min_ascending",
+            "round_to_4_via_case",
+            "round_to_2",
+            "round_to_3",
+            "round_to_4",
             "rlike",
             "to_lower",
             "to_lower_ords",
@@ -240,6 +247,65 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
+            case "round_to_4_via_case" -> {
+                FieldAttribute f = longField();
+
+                Expression ltkb = new LessThan(Source.EMPTY, f, kb());
+                Expression ltmb = new LessThan(Source.EMPTY, f, mb());
+                Expression ltgb = new LessThan(Source.EMPTY, f, gb());
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new Case(Source.EMPTY, ltkb, List.of(b(), ltmb, kb(), ltgb, mb(), gb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "CaseLazyEvaluator";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
+            case "round_to_2" -> {
+                FieldAttribute f = longField();
+
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new RoundTo(Source.EMPTY, f, List.of(b(), kb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "RoundToLong2";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
+            case "round_to_3" -> {
+                FieldAttribute f = longField();
+
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new RoundTo(Source.EMPTY, f, List.of(b(), kb(), mb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "RoundToLong3";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
+            case "round_to_4" -> {
+                FieldAttribute f = longField();
+
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new RoundTo(Source.EMPTY, f, List.of(b(), kb(), mb(), gb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "RoundToLong4";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
             case "to_lower", "to_lower_ords" -> {
                 FieldAttribute keywordField = keywordField();
                 ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
@@ -419,6 +485,69 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "round_to_4_via_case", "round_to_4" -> {
+                long b = 1;
+                long kb = ByteSizeUnit.KB.toBytes(1);
+                long mb = ByteSizeUnit.MB.toBytes(1);
+                long gb = ByteSizeUnit.GB.toBytes(1);
+
+                LongVector f = actual.<LongBlock>getBlock(0).asVector();
+                LongVector result = actual.<LongBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    long expected = f.getLong(i);
+                    if (expected < kb) {
+                        expected = b;
+                    } else if (expected < mb) {
+                        expected = kb;
+                    } else if (expected < gb) {
+                        expected = mb;
+                    } else {
+                        expected = gb;
+                    }
+                    if (result.getLong(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+                    }
+                }
+            }
+            case "round_to_3" -> {
+                long b = 1;
+                long kb = ByteSizeUnit.KB.toBytes(1);
+                long mb = ByteSizeUnit.MB.toBytes(1);
+
+                LongVector f = actual.<LongBlock>getBlock(0).asVector();
+                LongVector result = actual.<LongBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    long expected = f.getLong(i);
+                    if (expected < kb) {
+                        expected = b;
+                    } else if (expected < mb) {
+                        expected = kb;
+                    } else {
+                        expected = mb;
+                    }
+                    if (result.getLong(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+                    }
+                }
+            }
+            case "round_to_2" -> {
+                long b = 1;
+                long kb = ByteSizeUnit.KB.toBytes(1);
+
+                LongVector f = actual.<LongBlock>getBlock(0).asVector();
+                LongVector result = actual.<LongBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    long expected = f.getLong(i);
+                    if (expected < kb) {
+                        expected = b;
+                    } else {
+                        expected = kb;
+                    }
+                    if (result.getLong(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+                    }
+                }
+            }
             case "to_lower" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
             case "to_lower_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
             case "to_upper" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
@@ -450,7 +579,7 @@ private static void checkBytes(String operation, Page actual, boolean expectOrds
 
     private static Page page(String operation) {
         return switch (operation) {
-            case "abs", "add", "date_trunc", "equal_to_const" -> {
+            case "abs", "add", "date_trunc", "equal_to_const", "round_to_4_via_case", "round_to_2", "round_to_3", "round_to_4" -> {
                 var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
                 for (int i = 0; i < BLOCK_LENGTH; i++) {
                     builder.appendLong(i * 100_000);
@@ -540,6 +669,26 @@ private static Page page(String operation) {
         };
     }
 
+    private static Literal b() {
+        return lit(1L);
+    }
+
+    private static Literal kb() {
+        return lit(ByteSizeUnit.KB.toBytes(1));
+    }
+
+    private static Literal mb() {
+        return lit(ByteSizeUnit.MB.toBytes(1));
+    }
+
+    private static Literal gb() {
+        return lit(ByteSizeUnit.GB.toBytes(1));
+    }
+
+    private static Literal lit(long v) {
+        return new Literal(Source.EMPTY, v, DataType.LONG);
+    }
+
     @Benchmark
     @OperationsPerInvocation(1024 * BLOCK_LENGTH)
     public void run() {

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -30,8 +30,10 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.SourceFieldMetrics;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.plugins.PluginsLoader;
@@ -90,7 +92,7 @@ public class ScriptScoreBenchmark {
     private final SearchLookup lookup = new SearchLookup(
         fieldTypes::get,
         (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService),
-        SourceProvider.fromStoredFields()
+        SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP)
     );
 
     @Param({ "expression", "metal", "painless_cast", "painless_def" })

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,30 +14,27 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "dockerfiles/default/Dockerfile"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
 
     // Chainguard based wolfi image with latest jdk
-    // This is usually updated via renovatebot
-    // spotless:off
     WOLFI(
-        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
+        null,
         "-wolfi",
         "apk",
-        "Dockerfile"
+        "dockerfiles/wolfi/Dockerfile"
     ),
-    // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
     CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.ess"),
 
     CLOUD_ESS_FIPS(
-        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        null,
         "-cloud-ess-fips",
         "apk",
-        "Dockerfile.ess-fips"
+        "dockerfiles/cloud_ess_fips/Dockerfile"
     );
 
     private final String image;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -165,7 +165,7 @@ public void apply(Project project) {
             nonInputSystemProperties.systemProperty(TESTS_FEATURES_METADATA_PATH, () -> featureMetadataConfig.getAsPath());
 
             // Enable parallel execution for these tests since each test gets its own cluster
-            task.setMaxParallelForks(task.getProject().getGradle().getStartParameter().getMaxWorkerCount() / 2);
+            task.setMaxParallelForks(Math.max(1, task.getProject().getGradle().getStartParameter().getMaxWorkerCount() / 2));
             nonInputSystemProperties.systemProperty(TESTS_MAX_PARALLEL_FORKS_SYSPROP, () -> String.valueOf(task.getMaxParallelForks()));
 
             // Disable test failure reporting since this stuff is now captured in build scans

build-tools/build.gradle

@@ -51,6 +51,10 @@ gradlePlugin {
           id = 'elasticsearch.stable-esplugin'
           implementationClass = 'org.elasticsearch.gradle.plugin.StablePluginBuildPlugin'
         }
+        testBuildInfo {
+          id = 'elasticsearch.test-build-info'
+          implementationClass = 'org.elasticsearch.gradle.test.TestBuildInfoPlugin'
+        }
         javaRestTest {
             id = 'elasticsearch.java-rest-test'
             implementationClass = 'org.elasticsearch.gradle.test.JavaRestTestPlugin'