Introduce partitioning in TopN operator

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.3", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-micro-benchmarks.yml

@@ -0,0 +1,10 @@
+steps:
+  - label: periodic-micro-benchmarks
+    command: |
+      .ci/scripts/run-gradle.sh :benchmarks:run --args 'org.elasticsearch.benchmark._nightly -rf json -rff build/result.json'
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/periodic-packaging.template.yml

@@ -40,6 +40,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic-packaging.yml

@@ -319,8 +319,8 @@ steps:
         env:
           BWC_VERSION: 8.17.7
 
-      - label: "{{matrix.image}} / 8.18.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.2
+      - label: "{{matrix.image}} / 8.18.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -333,7 +333,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.2
+          BWC_VERSION: 8.18.3
 
       - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@@ -393,6 +393,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic-platform-support.yml

@@ -38,6 +38,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
             GRADLE_TASK:
               - checkPart1
               - checkPart2

.buildkite/pipelines/periodic.yml

@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.2#bwcTest
+      - label: 8.18.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.2
+          BWC_VERSION: 8.18.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.3", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -529,7 +529,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.7", "8.18.3", "8.19.0", "9.0.2", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/pull-request/packaging-tests-windows.yml

@@ -11,6 +11,7 @@ steps:
           setup:
             image:
               - windows-2022
+              - windows-2025
             PACKAGING_TASK:
               - default-windows-archive
         agents:

.ci/bwcVersions

@@ -17,7 +17,7 @@ BWC_VERSION:
   - "8.15.5"
   - "8.16.6"
   - "8.17.7"
-  - "8.18.2"
+  - "8.18.3"
   - "8.19.0"
   - "9.0.2"
   - "9.1.0"

.ci/scripts/run-gradle.sh

@@ -41,4 +41,4 @@ else
 fi
 
 set -e
-$GRADLEW -S --max-workers=$MAX_WORKERS $@
+$GRADLEW -S --max-workers=$MAX_WORKERS "$@"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
   - "8.17.7"
-  - "8.18.2"
+  - "8.18.3"
   - "8.19.0"
   - "9.0.2"
   - "9.1.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java → benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.esql;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
@@ -67,9 +67,9 @@ public class QueryPlanningBenchmark {
     }
 
     private PlanTelemetry telemetry;
-    private EsqlParser parser;
-    private Analyzer analyzer;
-    private LogicalPlanOptimizer optimizer;
+    private EsqlParser defaultParser;
+    private Analyzer manyFieldsAnalyzer;
+    private LogicalPlanOptimizer defaultOptimizer;
 
     @Setup
     public void setup() {
@@ -100,8 +100,8 @@ public void setup() {
         var functionRegistry = new EsqlFunctionRegistry();
 
         telemetry = new PlanTelemetry(functionRegistry);
-        parser = new EsqlParser();
-        analyzer = new Analyzer(
+        defaultParser = new EsqlParser();
+        manyFieldsAnalyzer = new Analyzer(
             new AnalyzerContext(
                 config,
                 functionRegistry,
@@ -112,10 +112,10 @@ public void setup() {
             ),
             new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
         );
-        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+        defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
     }
 
-    private LogicalPlan plan(String query) {
+    private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
         var parsed = parser.createStatement(query, new QueryParams(), telemetry);
         var analyzed = analyzer.analyze(parsed);
         var optimized = optimizer.optimize(analyzed);
@@ -124,6 +124,6 @@ private LogicalPlan plan(String query) {
 
     @Benchmark
     public void manyFields(Blackhole blackhole) {
-        blackhole.consume(plan("FROM test | LIMIT 10"));
+        blackhole.consume(plan(defaultParser, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
     }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -44,13 +45,15 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
+import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
-import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
 import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
+import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.session.Configuration;
@@ -128,6 +131,10 @@ static void selfTest() {
             "long_equal_to_int",
             "mv_min",
             "mv_min_ascending",
+            "round_to_4_via_case",
+            "round_to_2",
+            "round_to_3",
+            "round_to_4",
             "rlike",
             "to_lower",
             "to_lower_ords",
@@ -240,6 +247,65 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
                 yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
+            case "round_to_4_via_case" -> {
+                FieldAttribute f = longField();
+
+                Expression ltkb = new LessThan(Source.EMPTY, f, kb());
+                Expression ltmb = new LessThan(Source.EMPTY, f, mb());
+                Expression ltgb = new LessThan(Source.EMPTY, f, gb());
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new Case(Source.EMPTY, ltkb, List.of(b(), ltmb, kb(), ltgb, mb(), gb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "CaseLazyEvaluator";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
+            case "round_to_2" -> {
+                FieldAttribute f = longField();
+
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new RoundTo(Source.EMPTY, f, List.of(b(), kb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "RoundToLong2";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
+            case "round_to_3" -> {
+                FieldAttribute f = longField();
+
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new RoundTo(Source.EMPTY, f, List.of(b(), kb(), mb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "RoundToLong3";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
+            case "round_to_4" -> {
+                FieldAttribute f = longField();
+
+                EvalOperator.ExpressionEvaluator evaluator = EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
+                    new RoundTo(Source.EMPTY, f, List.of(b(), kb(), mb(), gb())),
+                    layout(f)
+                ).get(driverContext);
+                String desc = "RoundToLong4";
+                if (evaluator.toString().contains(desc) == false) {
+                    throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
+                }
+                yield evaluator;
+            }
             case "to_lower", "to_lower_ords" -> {
                 FieldAttribute keywordField = keywordField();
                 ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
@@ -419,6 +485,69 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "round_to_4_via_case", "round_to_4" -> {
+                long b = 1;
+                long kb = ByteSizeUnit.KB.toBytes(1);
+                long mb = ByteSizeUnit.MB.toBytes(1);
+                long gb = ByteSizeUnit.GB.toBytes(1);
+
+                LongVector f = actual.<LongBlock>getBlock(0).asVector();
+                LongVector result = actual.<LongBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    long expected = f.getLong(i);
+                    if (expected < kb) {
+                        expected = b;
+                    } else if (expected < mb) {
+                        expected = kb;
+                    } else if (expected < gb) {
+                        expected = mb;
+                    } else {
+                        expected = gb;
+                    }
+                    if (result.getLong(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+                    }
+                }
+            }
+            case "round_to_3" -> {
+                long b = 1;
+                long kb = ByteSizeUnit.KB.toBytes(1);
+                long mb = ByteSizeUnit.MB.toBytes(1);
+
+                LongVector f = actual.<LongBlock>getBlock(0).asVector();
+                LongVector result = actual.<LongBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    long expected = f.getLong(i);
+                    if (expected < kb) {
+                        expected = b;
+                    } else if (expected < mb) {
+                        expected = kb;
+                    } else {
+                        expected = mb;
+                    }
+                    if (result.getLong(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+                    }
+                }
+            }
+            case "round_to_2" -> {
+                long b = 1;
+                long kb = ByteSizeUnit.KB.toBytes(1);
+
+                LongVector f = actual.<LongBlock>getBlock(0).asVector();
+                LongVector result = actual.<LongBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    long expected = f.getLong(i);
+                    if (expected < kb) {
+                        expected = b;
+                    } else {
+                        expected = kb;
+                    }
+                    if (result.getLong(i) != expected) {
+                        throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+                    }
+                }
+            }
             case "to_lower" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
             case "to_lower_ords" -> checkBytes(operation, actual, true, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
             case "to_upper" -> checkBytes(operation, actual, false, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
@@ -450,7 +579,7 @@ private static void checkBytes(String operation, Page actual, boolean expectOrds
 
     private static Page page(String operation) {
         return switch (operation) {
-            case "abs", "add", "date_trunc", "equal_to_const" -> {
+            case "abs", "add", "date_trunc", "equal_to_const", "round_to_4_via_case", "round_to_2", "round_to_3", "round_to_4" -> {
                 var builder = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
                 for (int i = 0; i < BLOCK_LENGTH; i++) {
                     builder.appendLong(i * 100_000);
@@ -540,6 +669,26 @@ private static Page page(String operation) {
         };
     }
 
+    private static Literal b() {
+        return lit(1L);
+    }
+
+    private static Literal kb() {
+        return lit(ByteSizeUnit.KB.toBytes(1));
+    }
+
+    private static Literal mb() {
+        return lit(ByteSizeUnit.MB.toBytes(1));
+    }
+
+    private static Literal gb() {
+        return lit(ByteSizeUnit.GB.toBytes(1));
+    }
+
+    private static Literal lit(long v) {
+        return new Literal(Source.EMPTY, v, DataType.LONG);
+    }
+
     @Benchmark
     @OperationsPerInvocation(1024 * BLOCK_LENGTH)
     public void run() {