Merge remote-tracking branch 'origin/main' into lucene_snapshot_10_3

Author: romseygeek

.buildkite/pipelines/java-ea-check-new-build.yml

@@ -3,5 +3,5 @@ steps:
     env:
       RECENT_TIME_WINDOW: "24" # time window to consider a build as new in hours
     agents:
-      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.15"
       memory: "4G"

.buildkite/pipelines/pull-request-transport-versions.yml

@@ -1 +1,10 @@
-steps: []
+steps:
+  - label: update-transport-versions
+    command: '.buildkite/scripts/update-transport-versions.sh'
+    # New transport versions are always added via the main branch
+    if: build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == 'main'
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-16-32768
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/pr-upgrade.yml

@@ -2,5 +2,5 @@ steps:
   - label: pr-upgrade
     command: ".buildkite/scripts/run-pr-upgrade-tests.sh"
     agents:
-      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
+      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.15"
       memory: "4G"

.buildkite/pipelines/pull-request/validate-transport-version-backport.yml

@@ -0,0 +1,5 @@
+config:
+  skip-target-branches: "main"
+steps:
+  - label: validate-transport-version-backport
+    command: .buildkite/scripts/validate-transport-version-backport.sh

.buildkite/pull-requests.json

@@ -16,6 +16,25 @@
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
     },
+    {
+      "enabled": true,
+      "target_branch": "main",
+      "pipeline_slug": "elasticsearch-pull-request-transport-versions",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "allowed_list": ["elastic-renovate-prod[bot]"],
+      "set_commit_status": false,
+      "build_on_commit": true,
+      "build_on_comment": true,
+      "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
+      "retrigger_label_regex": "v[0-9]+\\.[0-9]+\\.[0-9]+",
+      "cancel_intermediate_builds": true,
+      "cancel_intermediate_builds_on_comment": false,
+      "skip_duplicate_builds": true
+    },
     {
       "enabled": true,
       "pipeline_slug": "elasticsearch-pull-request-performance-benchmark",

.buildkite/scripts/update-transport-versions.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+  echo "Not a pull request, skipping transport version update"
+  exit 0
+fi
+
+if ! git diff --exit-code; then
+  echo "Changes are present before updating transport versions, not running"
+  git status
+  exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Update transport version definitions"
+
+echo "--- Generating updated transport version definitions"
+# Calculate backport branches based on pull request version labels
+backport_branches=$(
+  echo "${GITHUB_PR_LABELS}" \
+    | tr ',' '\n' \
+    | grep -E "v[0-9]+\.[0-9]+\.[0-9]+" \
+    | sed -E 's/^v([0-9]+)\.([0-9]+)\.[0-9]+$/\1.\2/' \
+    | paste -sd, -
+)
+
+if [[ -z "${backport_branches}" ]]; then
+  echo "Skipping as pull request contains no version labels"
+  exit 0
+fi
+
+.ci/scripts/run-gradle.sh generateTransportVersion --backport-branches="${backport_branches}"
+
+if git diff --exit-code; then
+  echo "No changes found after updating transport versions. Don't need to auto commit."
+  exit 0
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email '[email protected]'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -A .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1

.buildkite/scripts/validate-transport-version-backport.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "--- Looking for transport version changes"
+
+# Get any changes in this pull request to transport definitions
+git fetch origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" --quiet
+changed_files=$(git diff --name-only "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" | grep -E "server/src/main/resources/transport/definitions/.*\.csv" || true)
+
+if [[ -z "${changed_files}" ]]; then
+  echo "No transport version changes detected."
+  exit 0
+fi
+
+# Compare those files against the main branch to ensure they are the same
+git fetch origin main --quiet
+while IFS= read -r file; do
+  if ! git diff --quiet origin/main -- "${file}"; then
+      echo "Changes to transport definition [${file}] missing from main branch."
+      echo "Transport changes must first be merged to main before being backported."
+      exit 1
+  fi
+done <<< "${changed_files}"
+
+echo "All transport changes exist in main branch."

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -86,7 +86,9 @@ public void setup() {
             false,
             Map.of(),
             System.nanoTime(),
-            false
+            false,
+            EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
         );
 
         var fields = 10_000;

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -365,7 +365,9 @@ private static Configuration configuration() {
             false,
             Map.of(),
             0,
-            false
+            false,
+            EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
+            EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY)
         );
     }
 

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -42,6 +42,7 @@ public class OptimizedScalarQuantizerBenchmark {
     int dims;
 
     float[] vector;
+    float[] scratch;
     float[] centroid;
     int[] destination;
 
@@ -57,6 +58,7 @@ public void init() {
         destination = new int[dims];
         vector = new float[dims];
         centroid = new float[dims];
+        scratch = new float[dims];
         for (int i = 0; i < dims; ++i) {
             vector[i] = random.nextFloat();
             centroid[i] = random.nextFloat();
@@ -65,14 +67,14 @@ public void init() {
 
     @Benchmark
     public int[] scalar() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
+        osq.scalarQuantize(vector, scratch, destination, bits, centroid);
         return destination;
     }
 
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
     public int[] vector() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
+        osq.scalarQuantize(vector, scratch, destination, bits, centroid);
         return destination;
     }
 }