Skip to content

Commit d350c14

Browse files
pawankartik-elasticpawankartik-elastic
authored
Merge branch 'main' into pkar/resolve-index-force-reconn
2 parents dcd2c14 + d84d141 commit d350c14

File tree

4,322 files changed

+330928
-272918
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

4,322 files changed

+330928
-272918
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 68 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -145,19 +145,80 @@ EOF
145145
EOF
146146
fi
147147

148+
# =============================================================================
149+
# PART 1: Test seed retrieval for ANY retry
150+
# =============================================================================
151+
# When a job is retried (regardless of SMART_RETRIES setting), retrieve and use
152+
# the same test seed from the original job to ensure reproducible test failures.
153+
# =============================================================================
154+
155+
# Initialize variables that may be reused by smart retry logic below
156+
BUILD_JSON=""
157+
ORIGIN_JOB_ID=""
158+
BUILD_SCAN_ID=""
159+
BUILD_SCAN_URL=""
160+
TESTS_SEED=""
161+
162+
if [[ "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
163+
echo "--- Retrieving test seed from original job"
164+
165+
if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
166+
if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
167+
168+
# Retrieve test seed from Buildkite metadata
169+
TESTS_SEED=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["tests-seed-" + $job_id]' 2>/dev/null)
170+
171+
# Retrieve build scan URL and ID for smart retry (reused in Part 2)
172+
BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null)
173+
BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
174+
175+
if [[ -n "$TESTS_SEED" ]] && [[ "$TESTS_SEED" != "null" ]]; then
176+
echo "Using test seed $TESTS_SEED from original job $ORIGIN_JOB_ID"
177+
export TESTS_SEED
178+
else
179+
echo "Warning: Could not find test seed in metadata for original job."
180+
echo "Tests will use a new random seed."
181+
TESTS_SEED=""
182+
fi
183+
else
184+
echo "Warning: Could not find origin job ID for retry."
185+
echo "Tests will use a new random seed."
186+
fi
187+
else
188+
echo "Warning: Failed to fetch build information from Buildkite API"
189+
echo "Tests will use a new random seed."
190+
fi
191+
fi
192+
193+
# =============================================================================
194+
# PART 2: Smart retry test filtering (only when SMART_RETRIES=true)
195+
# =============================================================================
196+
# When smart retries are enabled, fetch the list of failed tests from the
197+
# original job and filter this retry to only run those tests.
198+
# =============================================================================
148199
if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
149200
echo "--- Resolving previously failed tests"
150201
SMART_RETRY_STATUS="disabled"
151202
SMART_RETRY_DETAILS=""
152203

153-
if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
154-
if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
204+
# Check if we already have the build info from seed retrieval above
205+
if [[ -z "$BUILD_JSON" ]]; then
206+
# Fetch build info if not already available (shouldn't happen, but be safe)
207+
BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null) || BUILD_JSON=""
208+
fi
155209

156-
# Attempt to retrieve build scan ID directly from metadata
157-
BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
210+
if [[ -n "$BUILD_JSON" ]]; then
211+
# Get origin job ID if not already available
212+
if [[ -z "$ORIGIN_JOB_ID" ]] || [[ "$ORIGIN_JOB_ID" == "null" ]]; then
213+
ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) || ORIGIN_JOB_ID=""
214+
fi
158215

159-
# Retrieve build scan URL for annotation
160-
BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null)
216+
if [[ -n "$ORIGIN_JOB_ID" ]] && [[ "$ORIGIN_JOB_ID" != "null" ]]; then
217+
# Get build scan ID if not already available
218+
if [[ -z "$BUILD_SCAN_ID" ]] || [[ "$BUILD_SCAN_ID" == "null" ]]; then
219+
BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null) || BUILD_SCAN_ID=""
220+
BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null) || BUILD_SCAN_URL=""
221+
fi
161222

162223
if [[ -n "$BUILD_SCAN_ID" ]] && [[ "$BUILD_SCAN_ID" != "null" ]]; then
163224
# Validate BUILD_SCAN_ID format to prevent injection attacks
@@ -171,53 +232,6 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
171232
DEVELOCITY_BASE_URL="${DEVELOCITY_BASE_URL:-https://gradle-enterprise.elastic.co}"
172233
DEVELOCITY_FAILED_TEST_API_URL="${DEVELOCITY_BASE_URL}/api/tests/build/${BUILD_SCAN_ID}?testOutcomes=failed"
173234

174-
# Fetch test seed from build scan custom values
175-
# Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
176-
API_KEY="${DEVELOCITY_API_KEY:-$DEVELOCITY_API_ACCESS_KEY}"
177-
178-
if [[ -z "$API_KEY" ]]; then
179-
echo "Warning: No Develocity API key available (DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY)"
180-
echo "Test seed retrieval will be skipped"
181-
else
182-
DEVELOCITY_BUILD_SCAN_API_URL="${DEVELOCITY_BASE_URL}/api/builds/${BUILD_SCAN_ID}?models=gradle-attributes"
183-
TESTS_SEED=""
184-
185-
echo "Fetching test seed from build scan: $BUILD_SCAN_ID"
186-
echo "API URL: $DEVELOCITY_BUILD_SCAN_API_URL"
187-
188-
if BUILD_SCAN_DATA=$(curl --silent --show-error --compressed --request GET \
189-
--url "$DEVELOCITY_BUILD_SCAN_API_URL" \
190-
--max-time 30 \
191-
--header 'accept: application/json' \
192-
--header "authorization: Bearer $API_KEY" \
193-
--header 'content-type: application/json' 2>&1); then
194-
195-
# Check if we got valid JSON
196-
if echo "$BUILD_SCAN_DATA" | jq empty 2>/dev/null; then
197-
# Extract test seed from gradle attributes
198-
TESTS_SEED=$(printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | select(.name == "tests.seed") | .value' 2>/dev/null)
199-
200-
if [[ -z "$TESTS_SEED" ]] || [[ "$TESTS_SEED" == "null" ]]; then
201-
echo "Could not retrieve test seed from build scan"
202-
echo "Debug: Checking available gradle attributes..."
203-
printf '%s\n' "$BUILD_SCAN_DATA" | jq -r '.models.gradleAttributes.model.values[]? | .name' 2>/dev/null || echo "No gradle attributes found"
204-
TESTS_SEED=""
205-
else
206-
echo "Retrieved test seed: $TESTS_SEED"
207-
export TESTS_SEED
208-
fi
209-
else
210-
echo "Error: Invalid JSON response from Develocity API"
211-
echo "Response preview: ${BUILD_SCAN_DATA:0:200}"
212-
TESTS_SEED=""
213-
fi
214-
else
215-
echo "Error: Failed to fetch build scan data from Develocity API"
216-
echo "Curl output: ${BUILD_SCAN_DATA:0:200}"
217-
TESTS_SEED=""
218-
fi
219-
fi
220-
221235
# Add random delay to prevent API rate limiting from parallel retries
222236
sleep $((RANDOM % 5))
223237

@@ -227,7 +241,7 @@ if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]];
227241
--max-time 30 \
228242
--header 'accept: application/json' \
229243
--header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
230-
--header 'content-type: application/json' 2>/dev/null | jq --arg testseed "$TESTS_SEED" '. + {testseed: $testseed}' &> .failed-test-history.json; then
244+
--header 'content-type: application/json' 2>/dev/null | jq --arg testseed "${TESTS_SEED:-}" '. + {testseed: $testseed}' &> .failed-test-history.json; then
231245

232246
# Set secure file permissions
233247
chmod 600 .failed-test-history.json

‎.buildkite/hooks/pre-command.bat‎

Lines changed: 77 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -28,28 +28,95 @@ for /f "delims=" %%i in ('vault read -field^=token secret/ci/elastic-elasticsear
2828

2929
bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
3030

31-
REM Smart retries implementation
31+
REM =============================================================================
32+
REM PART 1: Test seed retrieval for ANY retry
33+
REM =============================================================================
34+
REM When a job is retried (regardless of SMART_RETRIES setting), retrieve and use
35+
REM the same test seed from the original job to ensure reproducible test failures.
36+
REM =============================================================================
37+
set ORIGIN_JOB_ID=
38+
set BUILD_SCAN_ID=
39+
set BUILD_SCAN_URL=
40+
set TESTS_SEED=
41+
42+
if defined BUILDKITE_RETRY_COUNT (
43+
if %BUILDKITE_RETRY_COUNT% GTR 0 (
44+
echo --- Retrieving test seed from original job
45+
46+
REM Fetch build information from Buildkite API
47+
curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
48+
49+
if exist .build-info.json (
50+
REM Extract origin job ID
51+
for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
52+
53+
if defined ORIGIN_JOB_ID (
54+
if not "!ORIGIN_JOB_ID!"=="null" (
55+
REM Retrieve test seed directly from Buildkite metadata
56+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"tests-seed-\" + $job_id]" .build-info.json 2^>nul') do set TESTS_SEED=%%i
57+
58+
REM Retrieve build scan URL and ID for smart retry (reused in Part 2)
59+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
60+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
61+
62+
if defined TESTS_SEED (
63+
if not "!TESTS_SEED!"=="null" (
64+
echo Using test seed !TESTS_SEED! from original job !ORIGIN_JOB_ID!
65+
) else (
66+
echo Warning: Could not find test seed in metadata for original job.
67+
echo Tests will use a new random seed.
68+
set TESTS_SEED=
69+
)
70+
) else (
71+
echo Warning: Could not find test seed in metadata for original job.
72+
echo Tests will use a new random seed.
73+
)
74+
) else (
75+
echo Warning: Could not find origin job ID for retry.
76+
echo Tests will use a new random seed.
77+
)
78+
) else (
79+
echo Warning: Could not find origin job ID for retry.
80+
echo Tests will use a new random seed.
81+
)
82+
) else (
83+
echo Warning: Failed to fetch build information from Buildkite API
84+
echo Tests will use a new random seed.
85+
)
86+
)
87+
)
88+
89+
REM =============================================================================
90+
REM PART 2: Smart retry test filtering (only when SMART_RETRIES=true)
91+
REM =============================================================================
92+
REM When smart retries are enabled, fetch the list of failed tests from the
93+
REM original job and filter this retry to only run those tests.
94+
REM =============================================================================
3295
if "%SMART_RETRIES%"=="true" (
3396
if defined BUILDKITE_RETRY_COUNT (
3497
if %BUILDKITE_RETRY_COUNT% GTR 0 (
3598
echo --- Resolving previously failed tests
3699
set SMART_RETRY_STATUS=disabled
37100
set SMART_RETRY_DETAILS=
38101

39-
REM Fetch build information from Buildkite API
40-
curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
102+
REM Check if we need to fetch build info (should already exist from Part 1)
103+
if not exist .build-info.json (
104+
curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
105+
)
41106

42107
if exist .build-info.json (
43-
REM Extract origin job ID
44-
for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
108+
REM Get origin job ID if not already set
109+
if not defined ORIGIN_JOB_ID (
110+
for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
111+
)
45112

46113
if defined ORIGIN_JOB_ID (
47114
if not "!ORIGIN_JOB_ID!"=="null" (
48-
REM Extract build scan ID directly (new way)
49-
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
50-
51-
REM Retrieve build scan URL for annotation
52-
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
115+
REM Get build scan ID if not already set
116+
if not defined BUILD_SCAN_ID (
117+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
118+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
119+
)
53120

54121
if defined BUILD_SCAN_ID (
55122
if not "!BUILD_SCAN_ID!"=="null" (
@@ -67,64 +134,6 @@ if "%SMART_RETRIES%"=="true" (
67134
if not defined DEVELOCITY_BASE_URL set DEVELOCITY_BASE_URL=https://gradle-enterprise.elastic.co
68135
set DEVELOCITY_FAILED_TEST_API_URL=!DEVELOCITY_BASE_URL!/api/tests/build/!BUILD_SCAN_ID!?testOutcomes=failed
69136

70-
REM Fetch test seed from build scan custom values
71-
REM Support both DEVELOCITY_API_KEY and DEVELOCITY_API_ACCESS_KEY
72-
set API_KEY=
73-
if defined DEVELOCITY_API_KEY (
74-
set API_KEY=%DEVELOCITY_API_KEY%
75-
) else if defined DEVELOCITY_API_ACCESS_KEY (
76-
set API_KEY=%DEVELOCITY_API_ACCESS_KEY%
77-
)
78-
79-
if not defined API_KEY (
80-
echo Warning: No Develocity API key available ^(DEVELOCITY_API_KEY or DEVELOCITY_API_ACCESS_KEY^)
81-
echo Test seed retrieval will be skipped
82-
set TESTS_SEED=
83-
) else (
84-
set DEVELOCITY_BUILD_SCAN_API_URL=!DEVELOCITY_BASE_URL!/api/builds/!BUILD_SCAN_ID!?models=gradle-attributes
85-
set TESTS_SEED=
86-
87-
echo Fetching test seed from build scan: !BUILD_SCAN_ID!
88-
echo API URL: !DEVELOCITY_BUILD_SCAN_API_URL!
89-
90-
REM Fetch build scan data
91-
curl --silent --show-error --compressed --request GET --url "!DEVELOCITY_BUILD_SCAN_API_URL!" --max-time 30 --header "accept: application/json" --header "authorization: Bearer !API_KEY!" --header "content-type: application/json" 2>nul | jq "." > .build-scan-data.json 2>nul
92-
93-
if exist .build-scan-data.json (
94-
REM Validate JSON
95-
jq empty .build-scan-data.json 2>nul
96-
if !errorlevel! equ 0 (
97-
REM Extract test seed from gradle attributes
98-
for /f "delims=" %%i in ('jq -r ".models.gradleAttributes.model.values[]? | select(.name == \"tests.seed\") | .value" .build-scan-data.json 2^>nul') do set TESTS_SEED=%%i
99-
100-
if defined TESTS_SEED (
101-
if not "!TESTS_SEED!"=="null" (
102-
echo Retrieved test seed: !TESTS_SEED!
103-
) else (
104-
echo Could not retrieve test seed from build scan
105-
echo Debug: Checking available gradle attributes...
106-
jq -r ".models.gradleAttributes.model.values[]? | .name" .build-scan-data.json 2>nul
107-
set TESTS_SEED=
108-
)
109-
) else (
110-
echo Could not retrieve test seed from build scan
111-
echo Debug: Checking available gradle attributes...
112-
jq -r ".models.gradleAttributes.model.values[]? | .name" .build-scan-data.json 2>nul
113-
set TESTS_SEED=
114-
)
115-
) else (
116-
echo Error: Invalid JSON response from Develocity API
117-
type .build-scan-data.json 2>nul | findstr /C:"^" | more +0 +10
118-
set TESTS_SEED=
119-
)
120-
121-
del .build-scan-data.json 2>nul
122-
) else (
123-
echo Error: Failed to fetch build scan data from Develocity API
124-
set TESTS_SEED=
125-
)
126-
)
127-
128137
REM Add random delay to prevent API rate limiting (0-4 seconds)
129138
set /a "delay=%RANDOM% %% 5"
130139
timeout /t !delay! /nobreak >nul 2>&1

‎.buildkite/pipelines/intake.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ steps:
6565
timeout_in_minutes: 300
6666
matrix:
6767
setup:
68-
BWC_VERSION: ["8.19.12", "9.2.6", "9.3.1", "9.4.0"]
68+
BWC_VERSION: ["8.19.13", "9.2.7", "9.3.2", "9.4.0"]
6969
agents:
7070
provider: gcp
7171
image: family/elasticsearch-ubuntu-2404

0 commit comments

Comments
 (0)