Merge branch 'main' into rn-test2

Author: lcawl

.buildkite/hooks/pre-command

@@ -152,7 +152,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
 
   if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
     if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
-      
+
       # Attempt to retrieve build scan ID directly from metadata
       BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
 
@@ -171,6 +171,53 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
           DEVELOCITY_BASE_URL="${DEVELOCITY_BASE_URL:-https://gradle-enterprise.elastic.co}"
           DEVELOCITY_FAILED_TEST_API_URL="${DEVELOCITY_BASE_URL}/api/tests/build/${BUILD_SCAN_ID}?testOutcomes=failed"
 
+          # Fetch test seed from build scan custom values
+          # Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
+          API_KEY="${DEVELOCITY_API_KEY:-$DEVELOCITY_API_ACCESS_KEY}"
+
+          if [[ -z "$API_KEY" ]]; then
+            echo "Warning: No Develocity API key available (DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY)"
+            echo "Test seed retrieval will be skipped"
+          else
+            DEVELOCITY_BUILD_SCAN_API_URL="${DEVELOCITY_BASE_URL}/api/builds/${BUILD_SCAN_ID}?models=gradle-attributes"
+            TESTS_SEED=""
+
+            echo "Fetching test seed from build scan: $BUILD_SCAN_ID"
+            echo "API URL: $DEVELOCITY_BUILD_SCAN_API_URL"
+
+            if BUILD_SCAN_DATA=$(curl --silent --show-error --compressed --request GET \
+              --url "$DEVELOCITY_BUILD_SCAN_API_URL" \
+              --max-time 30 \
+              --header 'accept: application/json' \
+              --header "authorization: Bearer $API_KEY" \
+              --header 'content-type: application/json' 2>&1); then
+
+              # Check if we got valid JSON
+              if echo "$BUILD_SCAN_DATA" | jq empty 2>/dev/null; then
+                # Extract test seed from gradle attributes
+                TESTS_SEED=$(printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | select(.name == "tests.seed") | .value' 2>/dev/null)
+
+                if [[ -z "$TESTS_SEED" ]] || [[ "$TESTS_SEED" == "null" ]]; then
+                  echo "Could not retrieve test seed from build scan"
+                  echo "Debug: Checking available gradle attributes..."
+                  printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | .name' 2>/dev/null || echo "No gradle attributes found"
+                  TESTS_SEED=""
+                else
+                  echo "Retrieved test seed: $TESTS_SEED"
+                  export TESTS_SEED
+                fi
+              else
+                echo "Error: Invalid JSON response from Develocity API"
+                echo "Response preview: ${BUILD_SCAN_DATA:0:200}"
+                TESTS_SEED=""
+              fi
+            else
+              echo "Error: Failed to fetch build scan data from Develocity API"
+              echo "Curl output: ${BUILD_SCAN_DATA:0:200}"
+              TESTS_SEED=""
+            fi
+          fi
+
           # Add random delay to prevent API rate limiting from parallel retries
           sleep $((RANDOM % 5))
 
@@ -180,7 +227,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
             --max-time 30 \
             --header 'accept: application/json' \
             --header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
-            --header 'content-type: application/json' 2>/dev/null | jq '.' &> .failed-test-history.json; then
+            --header 'content-type: application/json' 2>/dev/null | jq --arg testseed "$TESTS_SEED" '. + {testseed: $testseed}' &> .failed-test-history.json; then
 
             # Set secure file permissions
             chmod 600 .failed-test-history.json
@@ -196,7 +243,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
               ORIGIN_JOB_NAME="previous attempt"
             fi
 
-            echo "✓ Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
+            echo "Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
 
             # Create Buildkite annotation for visibility
             # Use unique context per job to support multiple retries

.buildkite/hooks/pre-command.bat

@@ -67,12 +67,70 @@ if "%SMART_RETRIES%"=="true" (
                   if not defined DEVELOCITY_BASE_URL set DEVELOCITY_BASE_URL=https://gradle-enterprise.elastic.co
                   set DEVELOCITY_FAILED_TEST_API_URL=!DEVELOCITY_BASE_URL!/api/tests/build/!BUILD_SCAN_ID!?testOutcomes=failed
 
+                  REM Fetch test seed from build scan custom values
+                  REM Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
+                  set API_KEY=
+                  if defined DEVELOCITY_API_KEY (
+                    set API_KEY=%DEVELOCITY_API_KEY%
+                  ) else if defined DEVELOCITY_API_ACCESS_KEY (
+                    set API_KEY=%DEVELOCITY_API_ACCESS_KEY%
+                  )
+
+                  if not defined API_KEY (
+                    echo Warning: No Develocity API key available ^(DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY^)
+                    echo Test seed retrieval will be skipped
+                    set TESTS_SEED=
+                  ) else (
+                    set DEVELOCITY_BUILD_SCAN_API_URL=!DEVELOCITY_BASE_URL!/api/builds/!BUILD_SCAN_ID!?models=gradle-attributes
+                    set TESTS_SEED=
+
+                    echo Fetching test seed from build scan: !BUILD_SCAN_ID!
+                    echo API URL: !DEVELOCITY_BUILD_SCAN_API_URL!
+
+                    REM Fetch build scan data
+                    curl --silent --show-error --compressed --request GET --url "!DEVELOCITY_BUILD_SCAN_API_URL!" --max-time 30 --header "accept: application/json" --header "authorization: Bearer !API_KEY!" --header "content-type: application/json" 2>nul | jq "." > .build-scan-data.json 2>nul
+
+                    if exist .build-scan-data.json (
+                      REM Validate JSON
+                      jq empty .build-scan-data.json 2>nul
+                      if !errorlevel! equ 0 (
+                        REM Extract test seed from gradle attributes
+                        for /f "delims=" %%i in ('jq -r ".models.gradleAttributes.model.values[]? | select(.name == \"tests.seed\") | .value" .build-scan-data.json 2^>nul') do set TESTS_SEED=%%i
+
+                        if defined TESTS_SEED (
+                          if not "!TESTS_SEED!"=="null" (
+                            echo Retrieved test seed: !TESTS_SEED!
+                          ) else (
+                            echo Could not retrieve test seed from build scan
+                            echo Debug: Checking available gradle attributes...
+                            jq -r ".models.gradleAttributes.model.values[]? | .name" .build-scan-data.json 2>nul
+                            set TESTS_SEED=
+                          )
+                        ) else (
+                          echo Could not retrieve test seed from build scan
+                          echo Debug: Checking available gradle attributes...
+                          jq -r ".models.gradleAttributes.model.values[]? | .name" .build-scan-data.json 2>nul
+                          set TESTS_SEED=
+                        )
+                      ) else (
+                        echo Error: Invalid JSON response from Develocity API
+                        type .build-scan-data.json 2>nul | findstr /C:"^" | more +0 +10
+                        set TESTS_SEED=
+                      )
+
+                      del .build-scan-data.json 2>nul
+                    ) else (
+                      echo Error: Failed to fetch build scan data from Develocity API
+                      set TESTS_SEED=
+                    )
+                  )
+
                   REM Add random delay to prevent API rate limiting (0-4 seconds)
                   set /a "delay=%RANDOM% %% 5"
                   timeout /t !delay! /nobreak >nul 2>&1
 
                   REM Fetch failed tests from Develocity API (curl will auto-decompress gzip with --compressed)
-                  curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq "." > .failed-test-history.json 2>nul
+                  curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq --arg testseed "!TESTS_SEED!" ". + {testseed: $testseed}" > .failed-test-history.json 2>nul
 
                   if exist .failed-test-history.json (
                     REM Set restrictive file permissions (owner only)
@@ -90,7 +148,7 @@ if "%SMART_RETRIES%"=="true" (
                     if not defined ORIGIN_JOB_NAME set ORIGIN_JOB_NAME=previous attempt
                     if "!ORIGIN_JOB_NAME!"=="null" set ORIGIN_JOB_NAME=previous attempt
 
-                    echo ✓ Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
+                    echo Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
 
                     REM Create Buildkite annotation for visibility
                     echo Rerunning failed build job [!ORIGIN_JOB_NAME!]^(!BUILD_SCAN_URL!^) > .smart-retry-annotation.txt
@@ -173,10 +231,11 @@ set "_DEVELOCITY_API_ACCESS_KEY=%DEVELOCITY_API_ACCESS_KEY%"
 set "_BUILDKITE_API_TOKEN=%BUILDKITE_API_TOKEN%"
 set "_JAVA_HOME=%JAVA_HOME%"
 set "_JAVA16_HOME=%JAVA16_HOME%"
+set "_TESTS_SEED=%TESTS_SEED%"
 
 REM End local scope and restore critical variables to parent environment
 REM This ensures bash scripts can access WORKSPACE, GRADLEW, and other variables
-ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%"
+ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%" && set "TESTS_SEED=%_TESTS_SEED%"
 
 bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.19.11", "9.1.11", "9.2.5", "9.3.0", "9.4.0"]
+            BWC_VERSION: ["8.19.12", "9.2.6", "9.4.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-java-ea.yml

@@ -366,8 +366,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.11 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.19.11#bwcTest
+      - label: 8.19.12 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.19.12#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -376,7 +376,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.11
+          BWC_VERSION: 8.19.12
         retry:
           automatic:
             - exit_status: "-1"
@@ -404,8 +404,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.11 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.11#bwcTest
+      - label: 9.1.10 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.1.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -414,7 +414,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
         retry:
           automatic:
             - exit_status: "-1"
@@ -423,8 +423,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.2.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.2.5#bwcTest
+      - label: 9.2.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.2.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -433,7 +433,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.2.5
+          BWC_VERSION: 9.2.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -557,7 +557,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.19.11", "9.1.11", "9.2.5", "9.3.0", "9.4.0"]
+            BWC_VERSION: ["8.19.12", "9.2.6", "9.4.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -336,8 +336,8 @@ steps:
         env:
           BWC_VERSION: 8.18.8
 
-      - label: "{{matrix.image}} / 8.19.11 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.11
+      - label: "{{matrix.image}} / 8.19.12 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.12
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -350,7 +350,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.11
+          BWC_VERSION: 8.19.12
 
       - label: "{{matrix.image}} / 9.0.8 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.8
@@ -368,8 +368,8 @@ steps:
         env:
           BWC_VERSION: 9.0.8
 
-      - label: "{{matrix.image}} / 9.1.11 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.11
+      - label: "{{matrix.image}} / 9.1.10 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.10
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -382,10 +382,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
 
-      - label: "{{matrix.image}} / 9.2.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.5
+      - label: "{{matrix.image}} / 9.2.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -398,7 +398,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.2.5
+          BWC_VERSION: 9.2.6
 
       - label: "{{matrix.image}} / 9.3.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.3.0

.buildkite/pipelines/periodic.yml

@@ -415,8 +415,8 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: 8.19.11 / bwc
-        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v8.19.11#bwcTest
+      - label: 8.19.12 / bwc
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v8.19.12#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -425,7 +425,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.11
+          BWC_VERSION: 8.19.12
         retry:
           automatic:
             - exit_status: "-1"
@@ -457,8 +457,8 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: 9.1.11 / bwc
-        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.11#bwcTest
+      - label: 9.1.10 / bwc
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.1.10#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -467,7 +467,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.11
+          BWC_VERSION: 9.1.10
         retry:
           automatic:
             - exit_status: "-1"
@@ -478,8 +478,8 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: 9.2.5 / bwc
-        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.2.5#bwcTest
+      - label: 9.2.6 / bwc
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.2.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -488,7 +488,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.2.5
+          BWC_VERSION: 9.2.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -625,7 +625,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.19.11", "9.1.11", "9.2.5", "9.3.0", "9.4.0"]
+            BWC_VERSION: ["8.19.12", "9.2.6", "9.4.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -692,7 +692,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.19.11", "9.1.11", "9.2.5", "9.3.0", "9.4.0"]
+            BWC_VERSION: ["8.19.12", "9.2.6", "9.4.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -749,7 +749,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk25
-            BWC_VERSION: ["8.19.11", "9.1.11", "9.2.5", "9.3.0", "9.4.0"]
+            BWC_VERSION: ["8.19.12", "9.2.6", "9.4.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404