Merge branch 'main' into disable_ml_on_macos_x86_64

Author: edsavage

.backportrc.json

@@ -1,9 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.13.0$" : "main",
+    "^v9.0.0$" : "main",
+    "^v8.17.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
-}
+}

.buildkite/hooks/pre-command

@@ -16,10 +16,10 @@ export COMPOSE_HTTP_TIMEOUT
 JOB_BRANCH="$BUILDKITE_BRANCH"
 export JOB_BRANCH
 
-GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+GRADLEW="./gradlew --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
 export GRADLEW
 
-GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+GRADLEW_BAT="./gradlew.bat --console=plain --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
 export GRADLEW_BAT
 
 export $(cat .ci/java-versions.properties | grep '=' | xargs)
@@ -78,11 +78,15 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
-  DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
-  export DOCKER_REGISTRY_USERNAME
+  if which docker > /dev/null 2>&1; then
+    DOCKER_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
+    export DOCKER_REGISTRY_USERNAME
 
-  DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
-  export DOCKER_REGISTRY_PASSWORD
+    DOCKER_REGISTRY_PASSWORD="$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/prod_docker_registry_credentials)"
+    export DOCKER_REGISTRY_PASSWORD
+
+    docker login --username "$DOCKER_REGISTRY_USERNAME" --password "$DOCKER_REGISTRY_PASSWORD" docker.elastic.co
+  fi
 fi
 
 if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
@@ -105,3 +109,11 @@ EOF
 <summary>Agent information from gobld</summary>
 EOF
 fi
+
+# Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2
+# We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way
+# And they fail. This sets up a manual entry for the hostname in dnsmasq.
+if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then
+  echo "$(hostname -i | cut -d' ' -f 2)  $(hostname -f)." | sudo tee /etc/dnsmasq.hosts
+  sudo systemctl restart dnsmasq.service
+fi

.buildkite/hooks/pre-command.bat

@@ -18,4 +18,6 @@ set JOB_BRANCH=%BUILDKITE_BRANCH%
 set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 
+bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
+
 exit /b 0

.buildkite/packer_cache.sh

@@ -29,6 +29,8 @@ for branch in "${branches[@]}"; do
   fi
 
   export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
-  "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
+  "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI --max-workers=4
+  "checkout/${branch}/gradlew" --stop
+  pkill -f '.*GradleDaemon.*'
   rm -rf "checkout/${branch}"
 done

.buildkite/pipelines/dra-workflow.yml

@@ -2,11 +2,14 @@ steps:
   - command: .buildkite/scripts/dra-workflow.sh
     env:
       USE_DRA_CREDENTIALS: "true"
+      USE_PROD_DOCKER_CREDENTIALS: "true"
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2204
       machineType: custom-32-98304
-      buildDirectory: /dev/shm/bk
+      localSsds: 1
+      localSsdInterface: nvme
+      diskSizeGb: 350
   - wait
   # The hadoop build depends on the ES artifact
   # So let's trigger the hadoop build any time we build a new staging artifact
@@ -16,4 +19,5 @@ steps:
       branch: "${BUILDKITE_BRANCH}"
       env:
         DRA_WORKFLOW: staging
+        USE_PROD_DOCKER_CREDENTIALS: "true"
     if: build.env('DRA_WORKFLOW') == 'staging'

.buildkite/pipelines/intake.template.yml

@@ -40,6 +40,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part5
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
@@ -67,6 +75,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
+    branches: "main 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -41,14 +41,22 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part5
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.17", "8.11.5", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["8.16.1", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -68,6 +76,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
+    branches: "main 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/lucene-snapshot/run-tests.yml

@@ -32,6 +32,22 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: custom-32-98304
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
+  - label: part5
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
@@ -40,7 +56,6 @@ steps:
         matrix:
           setup:
             BWC_VERSION:
-              - 7.17.13
               - 8.9.1
               - 8.10.0
         agents:

.buildkite/pipelines/periodic-packaging.template.yml

@@ -3,12 +3,10 @@ steps:
     steps:
       - label: "{{matrix.image}} / packaging-tests-unix"
         command: ./.ci/scripts/packaging-test.sh destructivePackagingTest
-        timeout_in_minutes: 300
+        timeout_in_minutes: 420
         matrix:
           setup:
             image:
-              - centos-7
-              - debian-10
               - debian-11
               - opensuse-leap-15
               - oraclelinux-7
@@ -28,8 +26,9 @@ steps:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
-          machineType: custom-16-32768
-        env: {}
+          machineType: n1-standard-8
+        env:
+          USE_PROD_DOCKER_CREDENTIALS: "true"
   - group: packaging-tests-upgrade
     steps: $BWC_STEPS
   - group: packaging-tests-windows
@@ -41,8 +40,6 @@ steps:
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
               - windows-2022
         agents:
           provider: gcp