Merge branch 'main' into pull/133314

Author: joegallo

.buildkite/pipelines/pull-request-performance-benchmark.yml

@@ -0,0 +1,3 @@
+steps:
+  - label: ":pipeline: Generate steps"
+    command: bash .buildkite/scripts/generate-pr-performance-benchmark.sh | buildkite-agent pipeline upload

.buildkite/pipelines/pull-request/performance-benchmark.yml

@@ -27,7 +27,8 @@
       "set_commit_status": false,
       "build_on_commit": false,
       "build_on_comment": true,
-      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
+      "target_branch": "main",
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\S+)( please)?$"
     }
   ]
 }

.buildkite/pull-requests.json

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
+env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
+merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
+
+buildkite-agent meta-data set pr_comment:custom-body:body \
+  "This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+buildkite-agent meta-data set pr_comment:custom-baseline:head \
+  "* Baseline: ${merge_base} (env ID ${env_id_baseline})"
+buildkite-agent meta-data set pr_comment:custom-contender:head \
+  "* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
+
+cat << _EOF_
+steps:
+  - label: Trigger baseline benchmark
+    trigger: elasticsearch-performance-esbench-pr
+    build:
+      message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
+      branch: master
+      env:
+        CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
+        ENV_ID: ${env_id_baseline}
+        REVISION: ${merge_base}
+  - label: Trigger contender benchmark
+    trigger: elasticsearch-performance-esbench-pr
+    build:
+      message: Contender benchmark for PR${GITHUB_PR_NUMBER}
+      branch: master
+      env:
+        CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
+        ENV_ID: ${env_id_contender}
+        ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
+        REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+  - wait: ~
+  - label: Modify PR comment
+    command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
+_EOF_

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -56,7 +56,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:jira");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:pagerduty");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search:qa:rest");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -323,7 +323,7 @@ spec:
       name: elasticsearch / pull-request / performance-benchmark
     spec:
       repository: elastic/elasticsearch
-      pipeline_file: .buildkite/pipelines/pull-request/performance-benchmark.yml
+      pipeline_file: .buildkite/pipelines/pull-request-performance-benchmark.yml
       env:
         ELASTIC_PR_COMMENTS_ENABLED: "true"
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"

catalog-info.yaml

@@ -110,7 +110,7 @@ RUN <%= retry.loop(package_manager,
           "      ${package_manager} update && \n" +
           "      ${package_manager} upgrade && \n" +
           "      ${package_manager} add --no-cache \n" +
-          "        bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd && \n" +
+          "        bash java-cacerts curl libstdc++ libsystemd netcat-openbsd p11-kit p11-kit-trust posix-libc-utils shadow tini unzip zip zstd wget && \n" +
           "      rm -rf /var/cache/apk/* "
      ) %>
 

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -354,10 +354,6 @@ tasks.register('buildDeb', Deb) {
   configure(commonDebConfig('x64'))
 }
 
-tasks.named('assemble'){
-  dependsOn 'buildDeb', 'buildAarch64Deb'
-}
-
 Closure commonRpmConfig(String architecture) {
   return {
     configure(commonPackageConfig('rpm', architecture))
@@ -391,11 +387,6 @@ tasks.register('buildRpm', Rpm) {
   configure(commonRpmConfig('x64'))
 }
 
-tasks.named('assemble'){
-  dependsOn 'buildRpm', 'buildAarch64Rpm'
-}
-
-
 Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }
 Closure rpmExists = { it -> new File('/bin/rpm').exists() || new File('/usr/bin/rpm').exists() || new File('/usr/local/bin/rpm').exists() }
 
@@ -409,6 +400,11 @@ subprojects {
 
   String buildTask = "build${it.name.replaceAll(/-[a-z]/) { it.substring(1).toUpperCase() }.capitalize()}"
   ext.buildDist = parent.tasks.named(buildTask)
+  tasks.named('assemble').configure {
+    dependsOn buildDist
+  }
+
+  // deprecated here for backwards compatibility of DistroTestPlugin and DistributionDownloadPlugin
   artifacts {
     'default' buildDist
   }

distribution/packages/build.gradle

@@ -0,0 +1,5 @@
+pr: 129693
+summary: Add top level normalizer for linear retriever
+area: Search
+type: enhancement
+issues: []

docs/changelog/129693.yaml

@@ -0,0 +1,16 @@
+pr: 130325
+summary: "Simulate API: Return 400 on invalid processor(s)"
+area: Ingest Node
+type: bug
+issues:
+ - 120731
+breaking:
+  title: Return 400 on invalid processor(s) in Simulate API
+  area: Ingest
+  details: "In earlier versions of {es}, the Simulate API would return a 500 error\
+    \ when encountering invalid processors. Now, it returns a 400 Bad Request error\
+    \ instead."
+  impact: Callers should expect a 400 Bad Request response when the Simulate API encounters
+    invalid processors. This change improves error handling and provides clearer feedback
+    on request issues.
+  notable: false