Merge branch 'main' into jackson221

Author: jdconrad

.buildkite/hooks/pre-command

@@ -152,7 +152,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
 
   if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
     if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
-      
+
       # Attempt to retrieve build scan ID directly from metadata
       BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
 
@@ -171,6 +171,53 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
           DEVELOCITY_BASE_URL="${DEVELOCITY_BASE_URL:-https://gradle-enterprise.elastic.co}"
           DEVELOCITY_FAILED_TEST_API_URL="${DEVELOCITY_BASE_URL}/api/tests/build/${BUILD_SCAN_ID}?testOutcomes=failed"
 
+          # Fetch test seed from build scan custom values
+          # Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
+          API_KEY="${DEVELOCITY_API_KEY:-$DEVELOCITY_API_ACCESS_KEY}"
+
+          if [[ -z "$API_KEY" ]]; then
+            echo "Warning: No Develocity API key available (DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY)"
+            echo "Test seed retrieval will be skipped"
+          else
+            DEVELOCITY_BUILD_SCAN_API_URL="${DEVELOCITY_BASE_URL}/api/builds/${BUILD_SCAN_ID}?models=gradle-attributes"
+            TESTS_SEED=""
+
+            echo "Fetching test seed from build scan: $BUILD_SCAN_ID"
+            echo "API URL: $DEVELOCITY_BUILD_SCAN_API_URL"
+
+            if BUILD_SCAN_DATA=$(curl --silent --show-error --compressed --request GET \
+              --url "$DEVELOCITY_BUILD_SCAN_API_URL" \
+              --max-time 30 \
+              --header 'accept: application/json' \
+              --header "authorization: Bearer $API_KEY" \
+              --header 'content-type: application/json' 2>&1); then
+
+              # Check if we got valid JSON
+              if echo "$BUILD_SCAN_DATA" | jq empty 2>/dev/null; then
+                # Extract test seed from gradle attributes
+                TESTS_SEED=$(printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | select(.name == "tests.seed") | .value' 2>/dev/null)
+
+                if [[ -z "$TESTS_SEED" ]] || [[ "$TESTS_SEED" == "null" ]]; then
+                  echo "Could not retrieve test seed from build scan"
+                  echo "Debug: Checking available gradle attributes..."
+                  printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | .name' 2>/dev/null || echo "No gradle attributes found"
+                  TESTS_SEED=""
+                else
+                  echo "Retrieved test seed: $TESTS_SEED"
+                  export TESTS_SEED
+                fi
+              else
+                echo "Error: Invalid JSON response from Develocity API"
+                echo "Response preview: ${BUILD_SCAN_DATA:0:200}"
+                TESTS_SEED=""
+              fi
+            else
+              echo "Error: Failed to fetch build scan data from Develocity API"
+              echo "Curl output: ${BUILD_SCAN_DATA:0:200}"
+              TESTS_SEED=""
+            fi
+          fi
+
           # Add random delay to prevent API rate limiting from parallel retries
           sleep $((RANDOM % 5))
 
@@ -180,7 +227,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
             --max-time 30 \
             --header 'accept: application/json' \
             --header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
-            --header 'content-type: application/json' 2>/dev/null | jq '.' &> .failed-test-history.json; then
+            --header 'content-type: application/json' 2>/dev/null | jq --arg testseed "$TESTS_SEED" '. + {testseed: $testseed}' &> .failed-test-history.json; then
 
             # Set secure file permissions
             chmod 600 .failed-test-history.json
@@ -196,7 +243,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
               ORIGIN_JOB_NAME="previous attempt"
             fi
 
-            echo "✓ Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
+            echo "Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
 
             # Create Buildkite annotation for visibility
             # Use unique context per job to support multiple retries

.buildkite/hooks/pre-command.bat

@@ -67,12 +67,70 @@ if "%SMART_RETRIES%"=="true" (
                   if not defined DEVELOCITY_BASE_URL set DEVELOCITY_BASE_URL=https://gradle-enterprise.elastic.co
                   set DEVELOCITY_FAILED_TEST_API_URL=!DEVELOCITY_BASE_URL!/api/tests/build/!BUILD_SCAN_ID!?testOutcomes=failed
 
+                  REM Fetch test seed from build scan custom values
+                  REM Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
+                  set API_KEY=
+                  if defined DEVELOCITY_API_KEY (
+                    set API_KEY=%DEVELOCITY_API_KEY%
+                  ) else if defined DEVELOCITY_API_ACCESS_KEY (
+                    set API_KEY=%DEVELOCITY_API_ACCESS_KEY%
+                  )
+
+                  if not defined API_KEY (
+                    echo Warning: No Develocity API key available ^(DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY^)
+                    echo Test seed retrieval will be skipped
+                    set TESTS_SEED=
+                  ) else (
+                    set DEVELOCITY_BUILD_SCAN_API_URL=!DEVELOCITY_BASE_URL!/api/builds/!BUILD_SCAN_ID!?models=gradle-attributes
+                    set TESTS_SEED=
+
+                    echo Fetching test seed from build scan: !BUILD_SCAN_ID!
+                    echo API URL: !DEVELOCITY_BUILD_SCAN_API_URL!
+
+                    REM Fetch build scan data
+                    curl --silent --show-error --compressed --request GET --url "!DEVELOCITY_BUILD_SCAN_API_URL!" --max-time 30 --header "accept: application/json" --header "authorization: Bearer !API_KEY!" --header "content-type: application/json" 2>nul | jq "." > .build-scan-data.json 2>nul
+
+                    if exist .build-scan-data.json (
+                      REM Validate JSON
+                      jq empty .build-scan-data.json 2>nul
+                      if !errorlevel! equ 0 (
+                        REM Extract test seed from gradle attributes
+                        for /f "delims=" %%i in ('jq -r ".models.gradleAttributes.model.values[]? | select(.name == \"tests.seed\") | .value" .build-scan-data.json 2^>nul') do set TESTS_SEED=%%i
+
+                        if defined TESTS_SEED (
+                          if not "!TESTS_SEED!"=="null" (
+                            echo Retrieved test seed: !TESTS_SEED!
+                          ) else (
+                            echo Could not retrieve test seed from build scan
+                            echo Debug: Checking available gradle attributes...
+                            jq -r ".models.gradleAttributes.model.values[]? | .name" .build-scan-data.json 2>nul
+                            set TESTS_SEED=
+                          )
+                        ) else (
+                          echo Could not retrieve test seed from build scan
+                          echo Debug: Checking available gradle attributes...
+                          jq -r ".models.gradleAttributes.model.values[]? | .name" .build-scan-data.json 2>nul
+                          set TESTS_SEED=
+                        )
+                      ) else (
+                        echo Error: Invalid JSON response from Develocity API
+                        type .build-scan-data.json 2>nul | findstr /C:"^" | more +0 +10
+                        set TESTS_SEED=
+                      )
+
+                      del .build-scan-data.json 2>nul
+                    ) else (
+                      echo Error: Failed to fetch build scan data from Develocity API
+                      set TESTS_SEED=
+                    )
+                  )
+
                   REM Add random delay to prevent API rate limiting (0-4 seconds)
                   set /a "delay=%RANDOM% %% 5"
                   timeout /t !delay! /nobreak >nul 2>&1
 
                   REM Fetch failed tests from Develocity API (curl will auto-decompress gzip with --compressed)
-                  curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq "." > .failed-test-history.json 2>nul
+                  curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq --arg testseed "!TESTS_SEED!" ". + {testseed: $testseed}" > .failed-test-history.json 2>nul
 
                   if exist .failed-test-history.json (
                     REM Set restrictive file permissions (owner only)
@@ -90,7 +148,7 @@ if "%SMART_RETRIES%"=="true" (
                     if not defined ORIGIN_JOB_NAME set ORIGIN_JOB_NAME=previous attempt
                     if "!ORIGIN_JOB_NAME!"=="null" set ORIGIN_JOB_NAME=previous attempt
 
-                    echo ✓ Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
+                    echo Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
 
                     REM Create Buildkite annotation for visibility
                     echo Rerunning failed build job [!ORIGIN_JOB_NAME!]^(!BUILD_SCAN_URL!^) > .smart-retry-annotation.txt
@@ -173,10 +231,11 @@ set "_DEVELOCITY_API_ACCESS_KEY=%DEVELOCITY_API_ACCESS_KEY%"
 set "_BUILDKITE_API_TOKEN=%BUILDKITE_API_TOKEN%"
 set "_JAVA_HOME=%JAVA_HOME%"
 set "_JAVA16_HOME=%JAVA16_HOME%"
+set "_TESTS_SEED=%TESTS_SEED%"
 
 REM End local scope and restore critical variables to parent environment
 REM This ensures bash scripts can access WORKSPACE, GRADLEW, and other variables
-ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%"
+ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%" && set "TESTS_SEED=%_TESTS_SEED%"
 
 bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 

.buildkite/pipelines/periodic-java-ea.yml

@@ -404,8 +404,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.11 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.11#bwcTest
+      - label: 9.1.10 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -414,7 +414,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
         retry:
           automatic:
             - exit_status: "-1"

.buildkite/pipelines/periodic-packaging.yml

@@ -368,8 +368,8 @@ steps:
         env:
           BWC_VERSION: 9.0.8
 
-      - label: "{{matrix.image}} / 9.1.11 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.11
+      - label: "{{matrix.image}} / 9.1.10 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.10
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -382,7 +382,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
 
       - label: "{{matrix.image}} / 9.2.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.6

.buildkite/pipelines/periodic.yml

@@ -457,8 +457,8 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: 9.1.11 / bwc
-        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.11#bwcTest
+      - label: 9.1.10 / bwc
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -467,7 +467,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
         retry:
           automatic:
             - exit_status: "-1"

.ci/bwcVersions

@@ -20,7 +20,7 @@ BWC_VERSION:
   - "8.18.8"
   - "8.19.12"
   - "9.0.8"
-  - "9.1.11"
+  - "9.1.10"
   - "9.2.6"
   - "9.3.0"
   - "9.4.0"

.ci/packer_cache.sh

@@ -49,6 +49,6 @@ for branch in "${branches[@]}"; do
   rm -rf checkout/$branch
   git clone --reference $(dirname "${SCRIPT}")/../.git https://github.com/elastic/elasticsearch.git --branch ${branch} --single-branch checkout/${branch}
   export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA}
-  ./checkout/${branch}/gradlew --project-dir ./checkout/${branch} --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none
+  ./checkout/${branch}/gradlew --project-dir ./checkout/${branch} --parallel clean -s resolveAllDependencies -Dbwc.throttle.maxParallelUsages=1 -Dorg.gradle.warning.mode=none
   rm -rf ./checkout/${branch}
 done

.ci/scripts/packaging-test.ps1

@@ -27,6 +27,18 @@ Remove-Item -Recurse -Force \tmp -ErrorAction Ignore
 New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
-& .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
+
+# Pass TESTS_SEED as Java system property if available
+$TestsSeedParam = ""
+if ($env:TESTS_SEED) {
+    $TestsSeedParam = "-Dtests.seed=$env:TESTS_SEED"
+    Write-Output "Using test seed: $env:TESTS_SEED"
+}
+
+if ($TestsSeedParam) {
+    & .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $TestsSeedParam $GradleTasks
+} else {
+    & .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
+}
 
 exit $LastExitCode

.ci/scripts/packaging-test.sh

@@ -77,11 +77,19 @@ git config --global --add safe.directory $WORKSPACE
 # sudo sets it's own PATH thus we use env to override that and call sudo annother time so we keep the secure root PATH
 # run with --continue to run both bats and java tests even if one fails
 # be explicit about Gradle home dir so we use the same even with sudo
+
+# Pass TESTS_SEED as Java system property if available
+TESTS_SEED_PARAM=""
+if [[ -n "${TESTS_SEED:-}" ]]; then
+  TESTS_SEED_PARAM="-Dtests.seed=$TESTS_SEED"
+  echo "Using test seed: $TESTS_SEED"
+fi
+
 sudo -E env \
   PATH=$BUILD_JAVA_HOME/bin:`sudo bash -c 'echo -n $PATH'` \
   --unset=ES_JAVA_HOME \
   --unset=JAVA_HOME \
   SYSTEM_JAVA_HOME=`readlink -f -n $BUILD_JAVA_HOME` \
   DOCKER_CONFIG="${HOME}/.docker" \
-  ./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ --continue $@
+  ./gradlew -g $HOME/.gradle --console=plain --scan --parallel --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ $TESTS_SEED_PARAM --continue $@
 

.ci/scripts/run-gradle.sh

@@ -41,4 +41,12 @@ else
 fi
 
 set -e
-$GRADLEW -S --max-workers=$MAX_WORKERS "$@"
+
+# Pass TESTS_SEED as Java system property if available
+TESTS_SEED_PARAM=""
+if [[ -n "${TESTS_SEED:-}" ]]; then
+  TESTS_SEED_PARAM="-Dtests.seed=$TESTS_SEED"
+  echo "Using test seed: $TESTS_SEED"
+fi
+
+$GRADLEW -S --max-workers=$MAX_WORKERS $TESTS_SEED_PARAM "$@"