Merge branch 'main' into feat/extend-kibana-system-osquery-permissions

Author: szwarckonrad

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.11", "8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -302,8 +302,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.10 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.10
+      - label: "{{matrix.image}} / 8.17.11 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.11
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -316,10 +316,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.10
+          BWC_VERSION: 8.17.11
 
-      - label: "{{matrix.image}} / 8.18.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.5
+      - label: "{{matrix.image}} / 8.18.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,10 +332,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.5
+          BWC_VERSION: 8.18.6
 
-      - label: "{{matrix.image}} / 8.19.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.1
+      - label: "{{matrix.image}} / 8.19.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,10 +348,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.1
+          BWC_VERSION: 8.19.3
 
-      - label: "{{matrix.image}} / 9.0.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.5
+      - label: "{{matrix.image}} / 9.0.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,10 +364,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.5
+          BWC_VERSION: 9.0.6
 
-      - label: "{{matrix.image}} / 9.1.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.1
+      - label: "{{matrix.image}} / 9.1.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -380,7 +380,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.1
+          BWC_VERSION: 9.1.3
 
       - label: "{{matrix.image}} / 9.2.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.0

.buildkite/pipelines/periodic.template.yml

@@ -246,7 +246,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.10 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.10#bwcTest
+      - label: 8.17.11 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.11#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.10
+          BWC_VERSION: 8.17.11
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.5#bwcTest
+      - label: 8.18.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.5
+          BWC_VERSION: 8.18.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.1#bwcTest
+      - label: 8.19.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.1
+          BWC_VERSION: 8.19.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.5#bwcTest
+      - label: 9.0.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.5
+          BWC_VERSION: 9.0.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -401,8 +401,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.1#bwcTest
+      - label: 9.1.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -411,7 +411,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.1
+          BWC_VERSION: 9.1.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.11", "8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.11", "8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -684,7 +684,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/pull-request/performance-benchmark.yml

@@ -0,0 +1,3 @@
+steps:
+  - label: ":pipeline: TODO"
+    command: echo TODO

.buildkite/pull-requests.json

@@ -15,6 +15,19 @@
       "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-pull-request-performance-benchmark",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": false,
+      "build_on_comment": true,
+      "trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
     }
   ]
 }

.ci/bwcVersions

@@ -16,9 +16,9 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.10"
-  - "8.18.5"
-  - "8.19.1"
-  - "9.0.5"
-  - "9.1.1"
+  - "8.17.11"
+  - "8.18.6"
+  - "8.19.3"
+  - "9.0.6"
+  - "9.1.3"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,7 @@
 BWC_VERSION:
-  - "8.17.10"
-  - "8.18.5"
-  - "8.19.1"
-  - "9.0.5"
-  - "9.1.1"
+  - "8.17.11"
+  - "8.18.6"
+  - "8.19.3"
+  - "9.0.6"
+  - "9.1.3"
   - "9.2.0"

.gitignore

@@ -69,7 +69,7 @@ testfixtures_shared/
 # Generated
 checkstyle_ide.xml
 x-pack/plugin/esql/src/main/generated-src/generated/
-server/src/main/resources/transport/defined/manifest.txt
+server/src/main/resources/transport/definitions/manifest.txt
 
 # JEnv
 .java-version

BUILDING.md

@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
-> [!Tip]  
+> [!Tip]
 > A manual confirmation of the Gradle generated checksums is currently not mandatory.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
@@ -186,6 +186,25 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
+### How do I test against java early access (ea) versions?
+
+Currently only openjdk EA builds by oracle are supported.
+To test against an early access version java version you can pass the major
+java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+
+```
+./gradlew clean test -Druntime.java=26-ea
+```
+
+This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+
+To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
+
+```
+./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+```
+
 #### How to use a Maven based third party dependency via `mavenlocal`?
 
 1. Clone the third party repository locally
@@ -229,7 +248,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
-> [!Note]  
+> [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 
@@ -261,7 +280,7 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
-> [!Note]  
+> [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository