Merge branch 'main' into remote-enrich-after-lookup-join

Author: smalyshev

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-java-ea.bwc.template.yml

@@ -0,0 +1,18 @@
+      - label: $BWC_VERSION / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=\$\$JAVA_EA_VERSION -Dbwc.checkout.align=true v$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: $BWC_VERSION
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3

.buildkite/pipelines/periodic-java-ea.template.yml

@@ -0,0 +1,206 @@
+env:
+  JAVA_EA_VERSION: "${JAVA_EA_VERSION:-25-pre}"
+
+steps:
+  - group: bwc
+    steps: $BWC_STEPS
+  - label: concurrent-search-tests
+    command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: encryption-at-rest
+    command: .buildkite/scripts/encryption-at-rest.sh -Druntime.java=$$JAVA_EA_VERSION
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: eql-correctness
+    command: .buildkite/scripts/eql-correctness.sh -Druntime.java=$$JAVA_EA_VERSION
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - group: java-matrix
+    steps:
+      - label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true $$GRADLE_TASK
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - ubuntu-2404
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - windows-2025
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          GRADLE_PARAMS: "-Druntime.java=$$JAVA_EA_VERSION"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.BWC_VERSION}} / matrix-bwc"
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION  -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            BWC_VERSION: $BWC_LIST
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
+  - label: release-tests
+    command: .buildkite/scripts/release-tests.sh
+    timeout_in_minutes: 360
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - label: single-processor-node-tests
+    command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true functionalTests
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      diskSizeGb: 350
+      machineType: custom-32-98304
+  - group: third-party tests
+    steps:
+      - label: third-party / azure-sas
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty-sas
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / azure
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / gcs
+        command: |
+          export google_storage_bucket=elasticsearch-ci-thirdparty
+          export google_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION gcsThirdPartyTest
+        env:
+          USE_3RD_PARTY_GCS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / geoip
+        command: |
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / s3
+        command: |
+          export amazon_s3_bucket=elasticsearch-ci.us-west-2
+          export amazon_s3_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION s3ThirdPartyTest
+        env:
+          USE_3RD_PARTY_S3_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / ms-graph
+        command: |
+          .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION msGraphThirdPartyTest
+        env:
+          USE_3RD_PARTY_MS_GRAPH_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+  - group: lucene-compat
+    steps:
+      - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=$$ES_VERSION -Dtests.bwc.refspec.main=$$ES_COMMIT luceneBwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            LUCENE_VERSION:
+              - "10.0.0"
+            ES_VERSION:
+              - "9.0.0"
+            ES_COMMIT:
+              - "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_VERSION: "{{matrix.ES_VERSION}}"
+          ES_COMMIT: "{{matrix.ES_COMMIT}}"