Merge branch 'main' into spacetime/whats-going-on

Author: masseyke

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.3", "9.2", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.1.0$" : "main",
+    "^v9.4.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/hooks/pre-command

@@ -44,7 +44,10 @@ export GRADLE_BUILD_CACHE_USERNAME
 GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
 export GRADLE_BUILD_CACHE_PASSWORD
 
-DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)"
+DEVELOCITY_API_ACCESS_KEY=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export DEVELOCITY_API_ACCESS_KEY
+
+DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$DEVELOCITY_API_ACCESS_KEY"
 export DEVELOCITY_ACCESS_KEY
 
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
@@ -64,6 +67,14 @@ if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   unset data
 fi
 
+if [[ "${USE_MAVEN_GPG:-}" == "true" ]]; then
+  vault_path="kv/ci-shared/release-eng/team-release-secrets/es-delivery/gpg"
+  ORG_GRADLE_PROJECT_signingKey=$(vault kv get --field="private_key" $vault_path)
+  ORG_GRADLE_PROJECT_signingPassword=$(vault kv get --field="passphase" $vault_path)
+  export ORG_GRADLE_PROJECT_signingKey
+  export ORG_GRADLE_PROJECT_signingPassword
+fi
+
 if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
   DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
   export DRA_VAULT_ROLE_ID_SECRET
@@ -104,6 +115,7 @@ if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
   export PERF_METRICS_PASSWORD
 fi
 
+
 # Authenticate to the Docker Hub public read-only registry
 if which docker > /dev/null 2>&1; then
   DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"
@@ -133,6 +145,155 @@ EOF
 EOF
 fi
 
+if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
+  echo "--- Resolving previously failed tests"
+  SMART_RETRY_STATUS="disabled"
+  SMART_RETRY_DETAILS=""
+
+  if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
+    if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
+
+      # Attempt to retrieve build scan ID directly from metadata
+      BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
+
+      # Retrieve build scan URL for annotation
+      BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null)
+
+      if [[ -n "$BUILD_SCAN_ID" ]] && [[ "$BUILD_SCAN_ID" != "null" ]]; then
+        # Validate BUILD_SCAN_ID format to prevent injection attacks
+        if [[ ! "$BUILD_SCAN_ID" =~ ^[a-zA-Z0-9_-]+$ ]]; then
+          echo "Smart Retry Configuration Issue"
+          echo "Invalid build scan ID format: $BUILD_SCAN_ID"
+          echo "Smart retry will be disabled for this run."
+          SMART_RETRY_STATUS="failed"
+          SMART_RETRY_DETAILS="Invalid build scan ID format"
+        else
+          DEVELOCITY_BASE_URL="${DEVELOCITY_BASE_URL:-https://gradle-enterprise.elastic.co}"
+          DEVELOCITY_FAILED_TEST_API_URL="${DEVELOCITY_BASE_URL}/api/tests/build/${BUILD_SCAN_ID}?testOutcomes=failed"
+
+          # Fetch test seed from build scan custom values
+          # Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
+          API_KEY="${DEVELOCITY_API_KEY:-$DEVELOCITY_API_ACCESS_KEY}"
+
+          if [[ -z "$API_KEY" ]]; then
+            echo "Warning: No Develocity API key available (DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY)"
+            echo "Test seed retrieval will be skipped"
+          else
+            DEVELOCITY_BUILD_SCAN_API_URL="${DEVELOCITY_BASE_URL}/api/builds/${BUILD_SCAN_ID}?models=gradle-attributes"
+            TESTS_SEED=""
+
+            echo "Fetching test seed from build scan: $BUILD_SCAN_ID"
+            echo "API URL: $DEVELOCITY_BUILD_SCAN_API_URL"
+
+            if BUILD_SCAN_DATA=$(curl --silent --show-error --compressed --request GET \
+              --url "$DEVELOCITY_BUILD_SCAN_API_URL" \
+              --max-time 30 \
+              --header 'accept: application/json' \
+              --header "authorization: Bearer $API_KEY" \
+              --header 'content-type: application/json' 2>&1); then
+
+              # Check if we got valid JSON
+              if echo "$BUILD_SCAN_DATA" | jq empty 2>/dev/null; then
+                # Extract test seed from gradle attributes
+                TESTS_SEED=$(printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | select(.name == "tests.seed") | .value' 2>/dev/null)
+
+                if [[ -z "$TESTS_SEED" ]] || [[ "$TESTS_SEED" == "null" ]]; then
+                  echo "Could not retrieve test seed from build scan"
+                  echo "Debug: Checking available gradle attributes..."
+                  printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | .name' 2>/dev/null || echo "No gradle attributes found"
+                  TESTS_SEED=""
+                else
+                  echo "Retrieved test seed: $TESTS_SEED"
+                  export TESTS_SEED
+                fi
+              else
+                echo "Error: Invalid JSON response from Develocity API"
+                echo "Response preview: ${BUILD_SCAN_DATA:0:200}"
+                TESTS_SEED=""
+              fi
+            else
+              echo "Error: Failed to fetch build scan data from Develocity API"
+              echo "Curl output: ${BUILD_SCAN_DATA:0:200}"
+              TESTS_SEED=""
+            fi
+          fi
+
+          # Add random delay to prevent API rate limiting from parallel retries
+          sleep $((RANDOM % 5))
+
+          if curl --compressed --request GET \
+            --url "$DEVELOCITY_FAILED_TEST_API_URL" \
+            --max-filesize 10485760 \
+            --max-time 30 \
+            --header 'accept: application/json' \
+            --header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
+            --header 'content-type: application/json' 2>/dev/null | jq --arg testseed "$TESTS_SEED" '. + {testseed: $testseed}' &> .failed-test-history.json; then
+
+            # Set secure file permissions
+            chmod 600 .failed-test-history.json
+
+            # Count filtered tests for visibility
+            FILTERED_WORK_UNITS=$(jq -r '.workUnits | length' .failed-test-history.json 2>/dev/null || echo "0")
+            SMART_RETRY_STATUS="enabled"
+            SMART_RETRY_DETAILS="Filtering to $FILTERED_WORK_UNITS work units with failures"
+
+            # Get the origin job name for better annotation labels
+            ORIGIN_JOB_NAME=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$ORIGIN_JOB_ID" '.jobs[] | select(.id == $jobId) | .name' 2>/dev/null)
+            if [ -z "$ORIGIN_JOB_NAME" ] || [ "$ORIGIN_JOB_NAME" = "null" ]; then
+              ORIGIN_JOB_NAME="previous attempt"
+            fi
+
+            echo "Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
+
+            # Create Buildkite annotation for visibility
+            # Use unique context per job to support multiple retries
+            cat << EOF | buildkite-agent annotate --style info --context "smart-retry-$BUILDKITE_JOB_ID"
+Rerunning failed build job [$ORIGIN_JOB_NAME]($BUILD_SCAN_URL)
+
+**Gradle Tasks with Failures:** $FILTERED_WORK_UNITS
+
+This retry will skip test tasks that had no failures in the previous run.
+EOF
+          else
+            echo "Smart Retry API Error"
+            echo "Failed to fetch failed tests from Develocity API"
+            echo "Smart retry will be disabled - all tests will run."
+            SMART_RETRY_STATUS="failed"
+            SMART_RETRY_DETAILS="API request failed"
+          fi
+        fi
+      else
+        echo "Smart Retry Configuration Issue"
+        echo "Could not find build scan ID in metadata."
+        echo "Smart retry will be disabled for this run."
+        SMART_RETRY_STATUS="failed"
+        SMART_RETRY_DETAILS="No build scan ID in metadata"
+      fi
+    else
+      echo "Smart Retry Configuration Issue"
+      echo "Could not find origin job ID for retry."
+      echo "Smart retry will be disabled for this run."
+      SMART_RETRY_STATUS="failed"
+      SMART_RETRY_DETAILS="No origin job ID found"
+    fi
+  else
+    echo "Smart Retry API Error"
+    echo "Failed to fetch build information from Buildkite API"
+    echo "Smart retry will be disabled - all tests will run."
+    SMART_RETRY_STATUS="failed"
+    SMART_RETRY_DETAILS="Buildkite API request failed"
+  fi
+
+  # Store metadata for tracking and analysis
+  buildkite-agent meta-data set "smart-retry-status" "$SMART_RETRY_STATUS" 2>/dev/null || true
+  if [[ -n "$SMART_RETRY_DETAILS" ]]; then
+    buildkite-agent meta-data set "smart-retry-details" "$SMART_RETRY_DETAILS" 2>/dev/null || true
+  fi
+  if [[ -n "$BUILD_SCAN_URL" ]]; then
+    buildkite-agent meta-data set "origin-build-scan" "$BUILD_SCAN_URL" 2>/dev/null || true
+  fi
+fi
+
 # Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2
 # We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way
 # And they fail. This sets up a manual entry for the hostname in dnsmasq.