Merge branch 'main' into lhirlimann-patch-3

Author: lhirlimann

.buildkite/pipelines/periodic-fwc.template.yml

@@ -1,6 +1,6 @@
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}}" / fwc
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -11,4 +11,4 @@ steps:
       setup:
         FWC_VERSION: $FWC_LIST
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-fwc.yml

@@ -1,7 +1,7 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}}" / fwc
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -12,4 +12,4 @@ steps:
       setup:
         FWC_VERSION: []
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/scripts/fwc-branches.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Configure FwC test branches
+# We do not want 7.x branch and only to run for branches that:
+# - have released at least one minor version (not main)
+# - have previous minor unreleased (not the oldest development branch)
+FWC_BRANCHES=()
+for branch in "${BRANCHES[@]}"; do
+  if [[ ! "$branch" =~ ^7\..* ]]; then
+    FWC_BRANCHES+=("$branch")
+  fi
+done
+# Remove first and last element
+FWC_BRANCHES=("${FWC_BRANCHES[@]:1:${#FWC_BRANCHES[@]}-2}")
+
+shouldRunFwcFor() {
+  local branch=$1
+  for fwc_branch in "${FWC_BRANCHES[@]}"; do
+    if [[ "$fwc_branch" == "$branch" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -2,39 +2,69 @@
 
 set -euo pipefail
 
+# uncomment for tests
+#function buildkite-agent {
+#  local command=$1
+#  echo "$@"
+#  if [ "$command" == "annotate" ]; then
+#    while read -r line; do
+#      echo "  read: $line";
+#    done
+#  fi
+#}
+
 env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
 env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
 merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
 
+# PR comment
+buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
 buildkite-agent meta-data set pr_comment:custom-body:body \
-  "This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+  "This build attempts two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR. \
+To estimate benchmark completion time inspect previous nightly runs [here](https://buildkite.com/elastic/elasticsearch-performance-esbench-nightly/builds?branch=master)."
 buildkite-agent meta-data set pr_comment:custom-baseline:head \
   "* Baseline: ${merge_base} (env ID ${env_id_baseline})"
 buildkite-agent meta-data set pr_comment:custom-contender:head \
   "* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
 
-cat << _EOF_
+# Buildkite annotation
+cat << _EOF1_ | buildkite-agent annotate --context "pr-benchmark-notification"
+  This build attempts two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of PR [${GITHUB_PR_NUMBER}](https://github.com/elastic/elasticsearch/pull/${GITHUB_PR_NUMBER}).
+  To estimate benchmark completion time inspect previous nightly runs [here](https://buildkite.com/elastic/elasticsearch-performance-esbench-nightly/builds?branch=master).
+  * Baseline: [${merge_base:0:7}](https://github.com/elastic/elasticsearch/commit/${merge_base}) (env ID ${env_id_baseline})
+  * Contender: [${GITHUB_PR_TRIGGERED_SHA:0:7}](https://github.com/elastic/elasticsearch/commit/${GITHUB_PR_TRIGGERED_SHA}) (env ID ${env_id_contender})
+_EOF1_
+
+cat << _EOF2_
 steps:
-  - label: Trigger baseline benchmark
+  - label: Trigger baseline benchmark with ${merge_base:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
-      message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
+      message: Baseline benchmark for PR ${GITHUB_PR_NUMBER} with ${merge_base:0:7}
       branch: master
       env:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
-  - label: Trigger contender benchmark
+  - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
-      message: Contender benchmark for PR${GITHUB_PR_NUMBER}
+      message: Contender benchmark for PR ${GITHUB_PR_NUMBER} with ${GITHUB_PR_TRIGGERED_SHA:0:7}
       branch: master
       env:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
   - wait: ~
-  - label: Modify PR comment
-    command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
-_EOF_
+  - label: Update PR comment and Buildkite annotation
+    command: |
+      buildkite-agent meta-data set pr_comment:custom-body:body "This build ran two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+      buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
+      cat << _EOF3_ | buildkite-agent annotate --context "pr-benchmark-notification"
+        This build ran two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of PR [${GITHUB_PR_NUMBER}](https://github.com/elastic/elasticsearch/pull/${GITHUB_PR_NUMBER}).
+        * Baseline: [${merge_base:0:7}](https://github.com/elastic/elasticsearch/commit/${merge_base}) (env ID ${env_id_baseline})
+        * Contender: [${GITHUB_PR_TRIGGERED_SHA:0:7}](https://github.com/elastic/elasticsearch/commit/${GITHUB_PR_TRIGGERED_SHA}) (env ID ${env_id_contender})
+        * [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)
+      _EOF3_
+_EOF2_

.buildkite/scripts/periodic.trigger.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 echo "steps:"
 
 source .buildkite/scripts/branches.sh
+source .buildkite/scripts/fwc-branches.sh
 
 IS_FIRST=true
 SKIP_DELAY="${SKIP_DELAY:-false}"
@@ -46,8 +47,7 @@ EOF
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
-# Include forward compatibility tests only for the bugfix branch
-if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then
+if shouldRunFwcFor "$BRANCH"; then
   cat <<EOF
   - trigger: elasticsearch-periodic-fwc
     label: Trigger periodic-fwc pipeline for $BRANCH

build-tools-internal/src/main/groovy/elasticsearch.fwc-test.gradle

@@ -7,19 +7,23 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-def fwcVersions = buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major &&  it.minor == VersionProperties.elasticsearchVersion.minor }
-def previousMinorSnapshot = buildParams.bwcVersions.unreleased.find { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor - 1 }
-
-fwcVersions.each { fwcVersion ->
-  tasks.register("v${fwcVersion}#fwcTest", StandaloneRestIntegTestTask) {
-    usesBwcDistribution(previousMinorSnapshot)
-    usesBwcDistribution(fwcVersion)
-    systemProperty("tests.old_cluster_version", previousMinorSnapshot)
-    systemProperty("tests.new_cluster_version", fwcVersion)
-    nonInputProperties.systemProperty 'tests.fwc', 'true'
+Version elasticsearchVersion = Version.fromString(versions.get("elasticsearch"))
+def fwcVersions = buildParams.bwcVersions.released.findAll { it.major == elasticsearchVersion.major &&  it.minor == elasticsearchVersion.minor }
+def targetMajor = elasticsearchVersion.minor > 0 ? elasticsearchVersion.major : elasticsearchVersion.major - 1
+def targetMinor = elasticsearchVersion.minor > 0 ? elasticsearchVersion.minor - 1 : buildParams.bwcVersions.unreleased.findAll { it.major == targetMajor }*.minor.max()
+def previousMinorSnapshot = buildParams.bwcVersions.unreleased.find { it.major == targetMajor && it.minor == targetMinor }
+if (previousMinorSnapshot != null) {
+  fwcVersions.each { fwcVersion ->
+    tasks.register("v${fwcVersion}#fwcTest", StandaloneRestIntegTestTask) {
+      usesBwcDistribution(previousMinorSnapshot)
+      usesBwcDistribution(fwcVersion)
+      systemProperty("tests.old_cluster_version", previousMinorSnapshot)
+      systemProperty("tests.new_cluster_version", fwcVersion)
+      nonInputProperties.systemProperty 'tests.fwc', 'true'
+    }
   }
 }
 

docs/changelog/131940.yaml

@@ -0,0 +1,5 @@
+pr: 131940
+summary: Allow remote enrich after LOOKUP JOIN
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/133616.yaml

@@ -0,0 +1,5 @@
+pr: 133616
+summary: Add mode filter to _resolve/index
+area: Indices APIs
+type: enhancement
+issues: []

docs/changelog/133775.yaml

@@ -0,0 +1,5 @@
+pr: 133775
+summary: Remove Transfer-Encoding from HTTP request with no content
+area: Network
+type: bug
+issues: []

docs/reference/elasticsearch/roles.md

@@ -1,20 +1,31 @@
 ---
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-roles.html
+applies_to:
+  stack: all
 ---
 
 # Roles [built-in-roles]
 
-:::{note}
-This section provides detailed **reference information** for Elasticsearch privileges.
+This section provides detailed **reference information** for {{es}} roles.
 
-Refer to [User roles](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/user-roles.md) in the **Deploy and manage** section for overview, getting started and conceptual information.
+To learn how to apply roles for {{stack}}, and to learn how role-based access control works, refer to [User roles](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/user-roles.md).
+
+
+:::{tip}
+{{serverless-full}} uses a different set of predefined roles. To learn more about the roles available in {{serverless-short}} projects, refer to [Elastic Cloud Serverless predefined roles](docs-content://deploy-manage/users-roles/cloud-organization/user-roles.md#general-assign-user-roles-table).
 :::
 
 The {{stack-security-features}} apply a default role to all users, including [anonymous users](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/anonymous-access.md). The default role enables users to access the authenticate endpoint, change their own passwords, and get information about themselves.
 
 There is also a set of built-in roles you can explicitly assign to users. These roles have a fixed set of privileges and cannot be updated.
 
+When you assign a user multiple roles, the user receives a union of the roles’ privileges.
+
+If the built-in roles do not address your use case, then you can create additional [custom roles](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/defining-roles.md).
+
+## Available roles [available-roles]
+
 $$$built-in-roles-apm-system$$$ `apm_system`
 :   Grants access necessary for the APM system user to send system-level data (such as monitoring) to {{es}}.
 
@@ -59,8 +70,8 @@ $$$built-in-roles-ingest-user$$$ `ingest_admin`
     ::::
 
 
-$$$built-in-roles-kibana-dashboard$$$ `kibana_dashboard_only_user`
-:   (This role is deprecated, please use [{{kib}} feature privileges](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/kibana-privileges.md#kibana-feature-privileges) instead). Grants read-only access to the {{kib}} Dashboard in every [space in {{kib}}](docs-content://deploy-manage/manage-spaces.md). This role does not have access to editing tools in {{kib}}.
+$$$built-in-roles-kibana-dashboard$$$ `kibana_dashboard_only_user` {applies_to}`stack: deprecated`
+:   This role is deprecated. Use [{{kib}} feature privileges](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/kibana-privileges.md#kibana-feature-privileges) instead. Grants read-only access to the {{kib}} Dashboard in every [space in {{kib}}](docs-content://deploy-manage/manage-spaces.md). This role does not have access to editing tools in {{kib}}.
 
 $$$built-in-roles-kibana-system$$$ `kibana_system`
 :   Grants access necessary for the {{kib}} system user to read from and write to the {{kib}} indices, manage index templates and tokens, and check the availability of the {{es}} cluster. It also permits activating, searching, and retrieving user profiles, as well as updating user profile data for the `kibana-*` namespace. This role grants read access to the `.monitoring-*` indices and read and write access to the `.reporting-*` indices. For more information, see [Configuring Security in {{kib}}](docs-content://deploy-manage/security/secure-your-cluster-deployment.md).
@@ -71,10 +82,12 @@ $$$built-in-roles-kibana-system$$$ `kibana_system`
 
 
 $$$built-in-roles-kibana-admin$$$ `kibana_admin`
-:   Grants access to all features in {{kib}}. For more information on {{kib}} authorization, see [Kibana authorization](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/built-in-roles.md).
+:   Grants access to all {{kib}} features in all spaces. For more information on {{kib}} authorization, see [Kibana authorization](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/built-in-roles.md).
+
+$$$built-in-roles-kibana-user$$$ `kibana_user` {applies_to}`stack: deprecated`
+:   This role is deprecated. Use the [`kibana_admin`](#built-in-roles-kibana-admin) role instead. Grants access to all features in {{kib}}.
 
-$$$built-in-roles-kibana-user$$$ `kibana_user`
-:   (This role is deprecated, please use the [`kibana_admin`](#built-in-roles-kibana-admin) role instead.) Grants access to all features in {{kib}}. For more information on {{kib}} authorization, see [Kibana authorization](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/built-in-roles.md).
+    For more information on {{kib}} authorization, see [Kibana authorization](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/built-in-roles.md).
 
 $$$built-in-roles-logstash-admin$$$ `logstash_admin`
 :   Grants access to the `.logstash*` indices for managing configurations, and grants necessary access for logstash-specific APIs exposed by the logstash x-pack plugin.
@@ -104,8 +117,10 @@ $$$built-in-roles-remote-monitoring-agent$$$ `remote_monitoring_agent`
 $$$built-in-roles-remote-monitoring-collector$$$ `remote_monitoring_collector`
 :   Grants the minimum privileges required to collect monitoring data for the {{stack}}.
 
-$$$built-in-roles-reporting-user$$$ `reporting_user`
-:   Grants the necessary privileges required to use {{report-features}} in {{kib}}, including generating and downloading reports. This role implicitly grants access to all Kibana reporting features, with each user having access only to their own reports. Note that reporting users should also be assigned additional roles that grant read access to the [indices](https://www.elastic.co/guide/en/elasticsearch/reference/current/defining-roles.html#roles-indices-priv) that will be used to generate reports.
+$$$built-in-roles-reporting-user$$$ `reporting_user` {applies_to}`stack: deprecated 9.0`
+:   This role is deprecated. Use [{{kib}} feature privileges](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/kibana-privileges.md#kibana-feature-privileges) instead.
+    
+    Grants the necessary privileges required to use {{report-features}} in {{kib}}, including generating and downloading reports. This role implicitly grants access to all {{kib}} reporting features, with each user having access only to their own reports. Note that reporting users should also be assigned additional roles that grant read access to the [indices](docs-content://deploy-manage/users-roles/cluster-or-deployment-auth/role-structure.md#roles-indices-priv) that will be used to generate reports.
 
 $$$built-in-roles-rollup-admin$$$ `rollup_admin`
 :   Grants `manage_rollup` cluster privileges, which enable you to manage and execute all rollup actions.