Merge remote-tracking branch 'origin/main' into pkar/resolve-index-force-reconn

Author: pawankartik-elastic

.buildkite/pipelines/periodic-java-ea.template.yml

@@ -1,5 +1,6 @@
 env:
   JAVA_EA_VERSION: "${JAVA_EA_VERSION:-26-pre}"
+  EXTRA_GRADLE_ARGS: "--continue"
 
 steps:
   - group: bwc

.buildkite/pipelines/periodic-java-ea.yml

@@ -1,6 +1,7 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-java-ea.template.yml
 env:
   JAVA_EA_VERSION: "${JAVA_EA_VERSION:-26-pre}"
+  EXTRA_GRADLE_ARGS: "--continue"
 
 steps:
   - group: bwc

.buildkite/pipelines/version-bump-pipeline.yml

@@ -0,0 +1,61 @@
+notify:
+  - slack:
+      channels:
+        - "#es-delivery-alerts"
+    if: (build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/) && (build.state == 'passed' || build.state == 'failed')
+  - slack:
+      channels:
+        - "#es-delivery-alerts"
+      message: |
+        🚦 Pipeline waiting for approval 🚦
+        Repo: `${REPO}`
+
+        Ready to fetch DRA artifacts - please unblock when ready.
+        New version: `${NEW_VERSION}`
+        Branch: `${BRANCH}`
+        Workflow: `${WORKFLOW}`
+        ${BUILDKITE_BUILD_URL}
+    if: build.state == "blocked"
+
+steps:
+  # TODO: replace this block step by real version bump logic
+  - block: "Ready to fetch for DRA artifacts?"
+    prompt: |
+      Unblock when your team is ready to proceed.
+
+      Trigger parameters:
+      - NEW_VERSION: ${NEW_VERSION}
+      - BRANCH: ${BRANCH}
+      - WORKFLOW: ${WORKFLOW}
+    key: block-get-dra-artifacts
+    blocked_state: running
+
+  - label: "Fetch DRA Artifacts"
+    key: fetch-dra-artifacts
+    depends_on: block-get-dra-artifacts
+    agents:
+      image: docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest
+      cpu: 250m
+      memory: 512Mi
+      ephemeralStorage: 1Gi
+    command:
+      - echo "Starting DRA artifacts retrieval..."
+    timeout_in_minutes: 240
+    retry:
+      automatic:
+        - exit_status: "*"
+          limit: 2
+      manual:
+        permit_on_passed: true
+
+    plugins:
+      - elastic/json-watcher#v1.0.0:
+          url: "https://artifacts-staging.elastic.co/elasticsearch/latest/${BRANCH}.json"
+          field: ".version"
+          expected_value: "${NEW_VERSION}"
+          polling_interval: "30"
+      - elastic/json-watcher#v1.0.0:
+          url: "https://storage.googleapis.com/elastic-artifacts-snapshot/elasticsearch/latest/${BRANCH}.json"
+          field: ".version"
+          expected_value: "${NEW_VERSION}-SNAPSHOT"
+          polling_interval: "30"

.buildkite/scripts/promql-compliance-bench.sh

@@ -13,6 +13,7 @@ readonly PIPELINE_FILE="${PIPELINE_FILE:-/tmp/${JOB_NAME}-pipeline.yml}"
 readonly REPO="${BENCH_REPO:-elastic/grafana-dashboards-analysis}"
 readonly REPO_DIR="${REPO_DIR:-/tmp/${JOB_NAME}-repo}"
 readonly INPUT_SUBDIR="${INPUT_SUBDIR:-results}"
+readonly INPUT_GLOB="${INPUT_GLOB:-*_raw_queries_simple.csv}"
 
 readonly GRADLE_TASK="${GRADLE_TASK:-:x-pack:plugin:esql:analyzePromqlQueries}"
 
@@ -69,9 +70,12 @@ find_input_files() {
 
   [[ -d "${root}" ]] || die "input directory does not exist: ${root}"
 
-  mapfile -d '' -t INPUT_FILES < <(find "${root}" -type f -name '*.csv' -print0 | sort -z)
+  # The analyzer expects one query per line in the form "<dashboardId>;<query>".
+  # The dashboards-analysis repo exports that shape as *_raw_queries_simple.csv;
+  # the other CSVs are summaries and spreadsheet-oriented reports.
+  mapfile -d '' -t INPUT_FILES < <(find "${root}" -type f -name "${INPUT_GLOB}" -print0 | sort -z)
 
-  (( ${#INPUT_FILES[@]} > 0 )) || die "no CSV files found in ${root}"
+  (( ${#INPUT_FILES[@]} > 0 )) || die "no CSV files matching ${INPUT_GLOB} found in ${root}"
 
   log "found ${#INPUT_FILES[@]} input files"
 }

.ci/scripts/run-gradle.sh

@@ -49,4 +49,4 @@ if [[ -n "${TESTS_SEED:-}" ]]; then
   echo "Using test seed: $TESTS_SEED"
 fi
 
-$GRADLEW -S --max-workers=$MAX_WORKERS $TESTS_SEED_PARAM "$@"
+$GRADLEW -S --max-workers=$MAX_WORKERS $TESTS_SEED_PARAM ${EXTRA_GRADLE_ARGS:-} "$@"

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.xpack.esql.inference.InferenceSettings;
 import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
+import org.elasticsearch.xpack.esql.parser.EsqlConfig;
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.QueryParams;
 import org.elasticsearch.xpack.esql.plan.IndexPattern;
@@ -77,6 +78,7 @@ public class QueryPlanningBenchmark {
     private Analyzer manyFieldsAnalyzer;
     private LogicalPlanOptimizer defaultOptimizer;
     private Configuration config;
+    private EsqlParser parser;
 
     @Setup
     public void setup() {
@@ -111,6 +113,7 @@ public void setup() {
         var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Set.of());
 
         var functionRegistry = new EsqlFunctionRegistry();
+        parser = new EsqlParser(new EsqlConfig(functionRegistry));
 
         // Assume all nodes are on the current version for the benchmark.
         TransportVersion minimumVersion = TransportVersion.current();
@@ -141,6 +144,6 @@ private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimi
 
     @Benchmark
     public void manyFields(Blackhole blackhole) {
-        blackhole.consume(plan(EsqlParser.INSTANCE, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
+        blackhole.consume(plan(parser, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
     }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ParallelParsingBenchmark.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.datasources.CloseableIterator;
 import org.elasticsearch.xpack.esql.datasources.ParallelParsingCoordinator;
+import org.elasticsearch.xpack.esql.datasources.spi.FormatReadContext;
 import org.elasticsearch.xpack.esql.datasources.spi.SegmentableFormatReader;
 import org.elasticsearch.xpack.esql.datasources.spi.SourceMetadata;
 import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
@@ -120,8 +121,7 @@ public void parallelParse(Blackhole bh) throws Exception {
                 List.of("line"),
                 1000,
                 parallelism,
-                executor,
-                SCHEMA
+                executor
             )
         ) {
             while (iter.hasNext()) {
@@ -161,30 +161,20 @@ public SourceMetadata metadata(StorageObject object) {
         }
 
         @Override
-        public CloseableIterator<Page> read(StorageObject object, List<String> projectedColumns, int batchSize) throws IOException {
-            return readSplit(object, projectedColumns, batchSize, false, true, SCHEMA);
-        }
-
-        @Override
-        public CloseableIterator<Page> readSplit(
-            StorageObject object,
-            List<String> projectedColumns,
-            int batchSize,
-            boolean skipFirstLine,
-            boolean lastSplit,
-            List<Attribute> resolvedAttributes
-        ) throws IOException {
+        public CloseableIterator<Page> read(StorageObject object, FormatReadContext context) throws IOException {
             final byte[] data;
             try (InputStream stream = object.newStream()) {
                 data = stream.readAllBytes();
             }
+            final int batchSize = context.batchSize();
+            final boolean skipFirstLine = context.firstSplit() == false;
 
             return new CloseableIterator<>() {
                 private int pos = 0;
                 private Page nextPage = null;
 
                 {
-                    // Skip first line if needed
+                    // Skip first line if needed (when not first split)
                     if (skipFirstLine) {
                         while (pos < data.length && data[pos] != '\n') {
                             pos++;

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/CsvErrorPolicyBenchmark.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.datasource.csv.CsvFormatReader;
 import org.elasticsearch.xpack.esql.datasources.CloseableIterator;
 import org.elasticsearch.xpack.esql.datasources.spi.ErrorPolicy;
+import org.elasticsearch.xpack.esql.datasources.spi.FormatReadContext;
 import org.elasticsearch.xpack.esql.datasources.spi.StorageObject;
 import org.elasticsearch.xpack.esql.datasources.spi.StoragePath;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -100,7 +101,7 @@ private int readAll(ErrorPolicy policy) throws IOException {
         CsvFormatReader reader = new CsvFormatReader(blockFactory);
         StorageObject obj = createStorageObject(csvData);
         int totalRows = 0;
-        try (CloseableIterator<Page> iter = reader.read(obj, null, 1000, policy)) {
+        try (CloseableIterator<Page> iter = reader.read(obj, FormatReadContext.builder().batchSize(1000).errorPolicy(policy).build())) {
             while (iter.hasNext()) {
                 totalRows += iter.next().getPositionCount();
             }

catalog-info.yaml

@@ -417,3 +417,36 @@ spec:
       provider_settings:
         build_pull_requests: true
         trigger_mode: none
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-version-bump
+  description: Buildkite Pipeline for elasticsearch version bump
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-version-bump
+
+spec:
+  type: buildkite-pipeline
+  owner: group:elasticsearch-team
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: Buildkite Pipeline for elasticsearch version bump
+      name: elasticsearch-version-bump
+    spec:
+      pipeline_file: ".buildkite/pipelines/version-bump-pipeline.yml"
+      provider_settings:
+        trigger_mode: none
+      repository: elastic/elasticsearch
+      teams:
+        elasticsearch-team: {}
+        ml-core: {}
+        everyone:
+          access_level: READ_ONLY
+        release-eng:
+          access_level: BUILD_AND_READ

docs/changelog/143381.yaml

@@ -0,0 +1,6 @@
+area: Downsampling
+issues:
+ - 136178
+pr: 143381
+summary: Aggregate counter downsampling preserves resets
+type: enhancement