Merge branch '8.19' into pkar/ccs-skip-un-test-leak-fix

Author: pawankartik-elastic

.backportrc.json

@@ -3,7 +3,7 @@
   "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.1.0$" : "main",
+    "^v8.19.4$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.30", "8.17.9", "8.18.4", "8.19.0"]
+            BWC_VERSION: ["7.17.30", "8.18.7", "8.19.4"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-fwc.template.yml

@@ -1,6 +1,6 @@
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -11,4 +11,4 @@ steps:
       setup:
         FWC_VERSION: $FWC_LIST
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-fwc.yml

@@ -1,7 +1,7 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -10,6 +10,6 @@ steps:
       buildDirectory: /dev/shm/bk
     matrix:
       setup:
-        FWC_VERSION: []
+        FWC_VERSION: ["8.19.0", "8.19.1", "8.19.2", "8.19.3"]
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-java-ea.bwc.template.yml

@@ -0,0 +1,18 @@
+      - label: $BWC_VERSION / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=\$\$JAVA_EA_VERSION -Dbwc.checkout.align=true v$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: $BWC_VERSION
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3

.buildkite/pipelines/periodic-java-ea.template.yml

@@ -0,0 +1,183 @@
+env:
+  JAVA_EA_VERSION: "${JAVA_EA_VERSION:-25-pre}"
+
+steps:
+  - group: bwc
+    steps: $BWC_STEPS
+  - label: concurrent-search-tests
+    command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: encryption-at-rest
+    command: .buildkite/scripts/encryption-at-rest.sh -Druntime.java=$$JAVA_EA_VERSION
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: eql-correctness
+    command: .buildkite/scripts/eql-correctness.sh -Druntime.java=$$JAVA_EA_VERSION
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - group: java-matrix
+    steps:
+      - label: "{{matrix.GRADLE_TASK}} / java-ea"
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true $$GRADLE_TASK
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - windows-2025
+              - ubuntu-2404
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.BWC_VERSION}} / matrix-bwc"
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION  -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            BWC_VERSION: $BWC_LIST
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: release-tests
+    command: .buildkite/scripts/release-tests.sh
+    timeout_in_minutes: 360
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: single-processor-node-tests
+    command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - group: third-party tests
+    steps:
+      - label: third-party / azure-sas
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty-sas
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / azure
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / gcs
+        command: |
+          export google_storage_bucket=elasticsearch-ci-thirdparty
+          export google_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION gcsThirdPartyTest
+        env:
+          USE_3RD_PARTY_GCS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / geoip
+        command: |
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / s3
+        command: |
+          export amazon_s3_bucket=elasticsearch-ci.us-west-2
+          export amazon_s3_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION s3ThirdPartyTest
+        env:
+          USE_3RD_PARTY_S3_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / ms-graph
+        command: |
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION msGraphThirdPartyTest
+        env:
+          USE_3RD_PARTY_MS_GRAPH_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+  - group: lucene-compat
+    steps:
+      - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=$$ES_VERSION -Dtests.bwc.refspec.main=$$ES_COMMIT luceneBwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            LUCENE_VERSION:
+              - "10.0.0"
+            ES_VERSION:
+              - "9.0.0"
+            ES_COMMIT:
+              - "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_VERSION: "{{matrix.ES_VERSION}}"
+          ES_COMMIT: "{{matrix.ES_COMMIT}}"