Skip to content

Commit 29fd7b8

Browse files
szybiaszybia
authored
Merge branch 'main' into put-settings-better-err-message
2 parents 4bcf3c8 + 1fc7e92 commit 29fd7b8

File tree

3,179 files changed

+98540
-36548
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

3,179 files changed

+98540
-36548
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@ export GRADLE_BUILD_CACHE_USERNAME
4444
GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
4545
export GRADLE_BUILD_CACHE_PASSWORD
4646

47-
DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)"
47+
DEVELOCITY_API_ACCESS_KEY=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
48+
export DEVELOCITY_API_ACCESS_KEY
49+
50+
DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$DEVELOCITY_API_ACCESS_KEY"
4851
export DEVELOCITY_ACCESS_KEY
4952

5053
BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
@@ -112,6 +115,7 @@ if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
112115
export PERF_METRICS_PASSWORD
113116
fi
114117

118+
115119
# Authenticate to the Docker Hub public read-only registry
116120
if which docker > /dev/null 2>&1; then
117121
DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"
@@ -141,6 +145,108 @@ EOF
141145
EOF
142146
fi
143147

148+
if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
149+
echo "--- Resolving previously failed tests"
150+
SMART_RETRY_STATUS="disabled"
151+
SMART_RETRY_DETAILS=""
152+
153+
if BUILD_JSON=$(curl --max-time 30 -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}?include_retried_jobs=true" 2>/dev/null); then
154+
if ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id' 2>/dev/null) && [ "$ORIGIN_JOB_ID" != "null" ] && [ -n "$ORIGIN_JOB_ID" ]; then
155+
156+
# Attempt to retrieve build scan ID directly from metadata
157+
BUILD_SCAN_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-id-" + $job_id]' 2>/dev/null)
158+
159+
# Retrieve build scan URL for annotation
160+
BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]' 2>/dev/null)
161+
162+
if [[ -n "$BUILD_SCAN_ID" ]] && [[ "$BUILD_SCAN_ID" != "null" ]]; then
163+
# Validate BUILD_SCAN_ID format to prevent injection attacks
164+
if [[ ! "$BUILD_SCAN_ID" =~ ^[a-zA-Z0-9_-]+$ ]]; then
165+
echo "Smart Retry Configuration Issue"
166+
echo "Invalid build scan ID format: $BUILD_SCAN_ID"
167+
echo "Smart retry will be disabled for this run."
168+
SMART_RETRY_STATUS="failed"
169+
SMART_RETRY_DETAILS="Invalid build scan ID format"
170+
else
171+
DEVELOCITY_BASE_URL="${DEVELOCITY_BASE_URL:-https://gradle-enterprise.elastic.co}"
172+
DEVELOCITY_FAILED_TEST_API_URL="${DEVELOCITY_BASE_URL}/api/tests/build/${BUILD_SCAN_ID}?testOutcomes=failed"
173+
174+
# Add random delay to prevent API rate limiting from parallel retries
175+
sleep $((RANDOM % 5))
176+
177+
if curl --compressed --request GET \
178+
--url "$DEVELOCITY_FAILED_TEST_API_URL" \
179+
--max-filesize 10485760 \
180+
--max-time 30 \
181+
--header 'accept: application/json' \
182+
--header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
183+
--header 'content-type: application/json' 2>/dev/null | jq '.' &> .failed-test-history.json; then
184+
185+
# Set secure file permissions
186+
chmod 600 .failed-test-history.json
187+
188+
# Count filtered tests for visibility
189+
FILTERED_WORK_UNITS=$(jq -r '.workUnits | length' .failed-test-history.json 2>/dev/null || echo "0")
190+
SMART_RETRY_STATUS="enabled"
191+
SMART_RETRY_DETAILS="Filtering to $FILTERED_WORK_UNITS work units with failures"
192+
193+
# Get the origin job name for better annotation labels
194+
ORIGIN_JOB_NAME=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg jobId "$ORIGIN_JOB_ID" '.jobs[] | select(.id == $jobId) | .name' 2>/dev/null)
195+
if [ -z "$ORIGIN_JOB_NAME" ] || [ "$ORIGIN_JOB_NAME" = "null" ]; then
196+
ORIGIN_JOB_NAME="previous attempt"
197+
fi
198+
199+
echo "✓ Smart retry enabled: filtering to $FILTERED_WORK_UNITS work units"
200+
201+
# Create Buildkite annotation for visibility
202+
# Use unique context per job to support multiple retries
203+
cat << EOF | buildkite-agent annotate --style info --context "smart-retry-$BUILDKITE_JOB_ID"
204+
Rerunning failed build job [$ORIGIN_JOB_NAME]($BUILD_SCAN_URL)
205+
206+
**Gradle Tasks with Failures:** $FILTERED_WORK_UNITS
207+
208+
This retry will skip test tasks that had no failures in the previous run.
209+
EOF
210+
else
211+
echo "Smart Retry API Error"
212+
echo "Failed to fetch failed tests from Develocity API"
213+
echo "Smart retry will be disabled - all tests will run."
214+
SMART_RETRY_STATUS="failed"
215+
SMART_RETRY_DETAILS="API request failed"
216+
fi
217+
fi
218+
else
219+
echo "Smart Retry Configuration Issue"
220+
echo "Could not find build scan ID in metadata."
221+
echo "Smart retry will be disabled for this run."
222+
SMART_RETRY_STATUS="failed"
223+
SMART_RETRY_DETAILS="No build scan ID in metadata"
224+
fi
225+
else
226+
echo "Smart Retry Configuration Issue"
227+
echo "Could not find origin job ID for retry."
228+
echo "Smart retry will be disabled for this run."
229+
SMART_RETRY_STATUS="failed"
230+
SMART_RETRY_DETAILS="No origin job ID found"
231+
fi
232+
else
233+
echo "Smart Retry API Error"
234+
echo "Failed to fetch build information from Buildkite API"
235+
echo "Smart retry will be disabled - all tests will run."
236+
SMART_RETRY_STATUS="failed"
237+
SMART_RETRY_DETAILS="Buildkite API request failed"
238+
fi
239+
240+
# Store metadata for tracking and analysis
241+
buildkite-agent meta-data set "smart-retry-status" "$SMART_RETRY_STATUS" 2>/dev/null || true
242+
if [[ -n "$SMART_RETRY_DETAILS" ]]; then
243+
buildkite-agent meta-data set "smart-retry-details" "$SMART_RETRY_DETAILS" 2>/dev/null || true
244+
fi
245+
if [[ -n "$BUILD_SCAN_URL" ]]; then
246+
buildkite-agent meta-data set "origin-build-scan" "$BUILD_SCAN_URL" 2>/dev/null || true
247+
fi
248+
fi
249+
144250
# Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2
145251
# We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way
146252
# And they fail. This sets up a manual entry for the hostname in dnsmasq.

‎.buildkite/hooks/pre-command.bat‎

Lines changed: 159 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
@ECHO OFF
2+
SETLOCAL EnableDelayedExpansion
23

34
FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i
45

@@ -17,10 +18,166 @@ set JOB_BRANCH=%BUILDKITE_BRANCH%
1718

1819
set GH_TOKEN=%VAULT_GITHUB_TOKEN%
1920

20-
set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
21-
set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
21+
REM Set credentials needed for gradle build cache and smart retries
22+
for /f "delims=" %%i in ('vault read -field^=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache') do set GRADLE_BUILD_CACHE_USERNAME=%%i
23+
for /f "delims=" %%i in ('vault read -field^=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache') do set GRADLE_BUILD_CACHE_PASSWORD=%%i
24+
for /f "delims=" %%i in ('vault read -field^=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache') do set DEVELOCITY_API_ACCESS_KEY=%%i
25+
set DEVELOCITY_ACCESS_KEY=gradle-enterprise.elastic.co=!DEVELOCITY_API_ACCESS_KEY!
26+
27+
for /f "delims=" %%i in ('vault read -field^=token secret/ci/elastic-elasticsearch/buildkite-api-token') do set BUILDKITE_API_TOKEN=%%i
2228

2329
bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
30+
31+
REM Smart retries implementation
32+
if "%SMART_RETRIES%"=="true" (
33+
if defined BUILDKITE_RETRY_COUNT (
34+
if %BUILDKITE_RETRY_COUNT% GTR 0 (
35+
echo --- Resolving previously failed tests
36+
set SMART_RETRY_STATUS=disabled
37+
set SMART_RETRY_DETAILS=
38+
39+
REM Fetch build information from Buildkite API
40+
curl --max-time 30 -H "Authorization: Bearer %BUILDKITE_API_TOKEN%" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/%BUILDKITE_PIPELINE_SLUG%/builds/%BUILDKITE_BUILD_NUMBER%?include_retried_jobs=true" -o .build-info.json 2>nul
41+
42+
if exist .build-info.json (
43+
REM Extract origin job ID
44+
for /f "delims=" %%i in ('jq -r --arg jobId "%BUILDKITE_JOB_ID%" ".jobs[] | select(.id == $jobId) | .retry_source.job_id" .build-info.json 2^>nul') do set ORIGIN_JOB_ID=%%i
45+
46+
if defined ORIGIN_JOB_ID (
47+
if not "!ORIGIN_JOB_ID!"=="null" (
48+
REM Extract build scan ID directly (new way)
49+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-id-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_ID=%%i
50+
51+
REM Retrieve build scan URL for annotation
52+
for /f "delims=" %%i in ('jq -r --arg job_id "!ORIGIN_JOB_ID!" ".meta_data[\"build-scan-\" + $job_id]" .build-info.json 2^>nul') do set BUILD_SCAN_URL=%%i
53+
54+
if defined BUILD_SCAN_ID (
55+
if not "!BUILD_SCAN_ID!"=="null" (
56+
57+
REM Validate using PowerShell (more reliable)
58+
powershell -NoProfile -Command "exit -not ('!BUILD_SCAN_ID!' -match '^[a-zA-Z0-9_\-]+$')"
59+
if errorlevel 1 (
60+
echo Smart Retry Configuration Issue
61+
echo Invalid build scan ID format: !BUILD_SCAN_ID!
62+
echo Smart retry will be disabled for this run.
63+
set SMART_RETRY_STATUS=failed
64+
set SMART_RETRY_DETAILS=Invalid build scan ID format
65+
) else (
66+
REM Set Develocity API URL
67+
if not defined DEVELOCITY_BASE_URL set DEVELOCITY_BASE_URL=https://gradle-enterprise.elastic.co
68+
set DEVELOCITY_FAILED_TEST_API_URL=!DEVELOCITY_BASE_URL!/api/tests/build/!BUILD_SCAN_ID!?testOutcomes=failed
69+
70+
REM Add random delay to prevent API rate limiting (0-4 seconds)
71+
set /a "delay=%RANDOM% %% 5"
72+
timeout /t !delay! /nobreak >nul 2>&1
73+
74+
REM Fetch failed tests from Develocity API (curl will auto-decompress gzip with --compressed)
75+
curl --compressed --request GET --url "!DEVELOCITY_FAILED_TEST_API_URL!" --max-filesize 10485760 --max-time 30 --header "accept: application/json" --header "authorization: Bearer %DEVELOCITY_API_ACCESS_KEY%" --header "content-type: application/json" 2>nul | jq "." > .failed-test-history.json 2>nul
76+
77+
if exist .failed-test-history.json (
78+
REM Set restrictive file permissions (owner only)
79+
icacls .failed-test-history.json /inheritance:r /grant:r "%USERNAME%:(R,W)" >nul 2>&1
80+
81+
REM Count filtered tests for visibility
82+
for /f "delims=" %%i in ('jq -r ".workUnits | length" .failed-test-history.json 2^>nul') do set FILTERED_WORK_UNITS=%%i
83+
if not defined FILTERED_WORK_UNITS set FILTERED_WORK_UNITS=0
84+
85+
set SMART_RETRY_STATUS=enabled
86+
set SMART_RETRY_DETAILS=Filtering to !FILTERED_WORK_UNITS! work units with failures
87+
88+
REM Get the origin job name for better annotation labels
89+
for /f "delims=" %%i in ('jq -r --arg jobId "!ORIGIN_JOB_ID!" ".jobs[] | select(.id == $jobId) | .name" .build-info.json 2^>nul') do set ORIGIN_JOB_NAME=%%i
90+
if not defined ORIGIN_JOB_NAME set ORIGIN_JOB_NAME=previous attempt
91+
if "!ORIGIN_JOB_NAME!"=="null" set ORIGIN_JOB_NAME=previous attempt
92+
93+
echo ✓ Smart retry enabled: filtering to !FILTERED_WORK_UNITS! work units
94+
95+
REM Create Buildkite annotation for visibility
96+
echo Rerunning failed build job [!ORIGIN_JOB_NAME!]^(!BUILD_SCAN_URL!^) > .smart-retry-annotation.txt
97+
echo. >> .smart-retry-annotation.txt
98+
echo **Gradle Tasks with Failures:** !FILTERED_WORK_UNITS! >> .smart-retry-annotation.txt
99+
echo. >> .smart-retry-annotation.txt
100+
echo This retry will skip test tasks that had no failures in the previous run. >> .smart-retry-annotation.txt
101+
buildkite-agent annotate --style info --context "smart-retry-!BUILDKITE_JOB_ID!" < .smart-retry-annotation.txt
102+
del .smart-retry-annotation.txt 2>nul
103+
) else (
104+
echo Smart Retry API Error
105+
echo Failed to fetch failed tests from Develocity API
106+
echo Smart retry will be disabled - all tests will run.
107+
set SMART_RETRY_STATUS=failed
108+
set SMART_RETRY_DETAILS=API request failed
109+
)
110+
)
111+
) else (
112+
echo Smart Retry Configuration Issue
113+
echo Could not find build scan ID in metadata.
114+
echo Smart retry will be disabled for this run.
115+
set SMART_RETRY_STATUS=failed
116+
set SMART_RETRY_DETAILS=No build scan ID in metadata
117+
)
118+
) else (
119+
echo Smart Retry Configuration Issue
120+
echo Could not find build scan ID in metadata.
121+
echo Smart retry will be disabled for this run.
122+
set SMART_RETRY_STATUS=failed
123+
set SMART_RETRY_DETAILS=No build scan ID in metadata
124+
)
125+
) else (
126+
echo Smart Retry Configuration Issue
127+
echo Could not find origin job ID for retry.
128+
echo Smart retry will be disabled for this run.
129+
set SMART_RETRY_STATUS=failed
130+
set SMART_RETRY_DETAILS=No origin job ID found
131+
)
132+
) else (
133+
echo Smart Retry Configuration Issue
134+
echo Could not find origin job ID for retry.
135+
echo Smart retry will be disabled for this run.
136+
set SMART_RETRY_STATUS=failed
137+
set SMART_RETRY_DETAILS=No origin job ID found
138+
)
139+
140+
REM Clean up temporary build info file
141+
del .build-info.json 2>nul
142+
) else (
143+
echo Smart Retry API Error
144+
echo Failed to fetch build information from Buildkite API
145+
echo Smart retry will be disabled - all tests will run.
146+
set SMART_RETRY_STATUS=failed
147+
set SMART_RETRY_DETAILS=Buildkite API request failed
148+
)
149+
150+
REM Store metadata for tracking and analysis
151+
buildkite-agent meta-data set "smart-retry-status" "!SMART_RETRY_STATUS!" 2>nul
152+
if defined SMART_RETRY_DETAILS (
153+
buildkite-agent meta-data set "smart-retry-details" "!SMART_RETRY_DETAILS!" 2>nul
154+
)
155+
if defined BUILD_SCAN_URL (
156+
buildkite-agent meta-data set "origin-build-scan" "!BUILD_SCAN_URL!" 2>nul
157+
)
158+
)
159+
)
160+
)
161+
162+
REM Capture variables before ENDLOCAL so they can be restored to parent scope
163+
set "_WORKSPACE=%WORKSPACE%"
164+
set "_GRADLEW=%GRADLEW%"
165+
set "_GRADLEW_BAT=%GRADLEW_BAT%"
166+
set "_BUILD_NUMBER=%BUILD_NUMBER%"
167+
set "_JOB_BRANCH=%JOB_BRANCH%"
168+
set "_GH_TOKEN=%GH_TOKEN%"
169+
set "_GRADLE_BUILD_CACHE_USERNAME=%GRADLE_BUILD_CACHE_USERNAME%"
170+
set "_GRADLE_BUILD_CACHE_PASSWORD=%GRADLE_BUILD_CACHE_PASSWORD%"
171+
set "_DEVELOCITY_ACCESS_KEY=%DEVELOCITY_ACCESS_KEY%"
172+
set "_DEVELOCITY_API_ACCESS_KEY=%DEVELOCITY_API_ACCESS_KEY%"
173+
set "_BUILDKITE_API_TOKEN=%BUILDKITE_API_TOKEN%"
174+
set "_JAVA_HOME=%JAVA_HOME%"
175+
set "_JAVA16_HOME=%JAVA16_HOME%"
176+
177+
REM End local scope and restore critical variables to parent environment
178+
REM This ensures bash scripts can access WORKSPACE, GRADLEW, and other variables
179+
ENDLOCAL && set "WORKSPACE=%_WORKSPACE%" && set "GRADLEW=%_GRADLEW%" && set "GRADLEW_BAT=%_GRADLEW_BAT%" && set "BUILD_NUMBER=%_BUILD_NUMBER%" && set "JOB_BRANCH=%_JOB_BRANCH%" && set "GH_TOKEN=%_GH_TOKEN%" && set "GRADLE_BUILD_CACHE_USERNAME=%_GRADLE_BUILD_CACHE_USERNAME%" && set "GRADLE_BUILD_CACHE_PASSWORD=%_GRADLE_BUILD_CACHE_PASSWORD%" && set "DEVELOCITY_ACCESS_KEY=%_DEVELOCITY_ACCESS_KEY%" && set "DEVELOCITY_API_ACCESS_KEY=%_DEVELOCITY_API_ACCESS_KEY%" && set "BUILDKITE_API_TOKEN=%_BUILDKITE_API_TOKEN%" && set "JAVA_HOME=%_JAVA_HOME%" && set "JAVA16_HOME=%_JAVA16_HOME%"
180+
24181
bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
25182

26183
exit /b 0

‎.buildkite/pipelines/cuvs-snapshot/run-tests.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ steps:
33
command: .buildkite/scripts/cuvs-snapshot/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints {{matrix.GRADLE_TASK}}
44
timeout_in_minutes: 300
55
agents:
6-
provider: gcp
7-
image: family/elasticsearch-ubuntu-2404-nvidia
8-
machineType: g2-standard-32
9-
buildDirectory: /dev/shm/bk
10-
zones: us-central1-b,us-central1-c
6+
provider: aws
7+
imagePrefix: elasticsearch-aws-ubuntu-2404-nvidia
8+
instanceType: g6.8xlarge
9+
diskSizeGb: 350
10+
diskType: gp3
11+
diskName: /dev/sda1
1112
env:
1213
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
1314
matrix:

‎.buildkite/pipelines/cuvs-snapshot/update-snapshot.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ steps:
22
- label: "Smoke test and update new cuVS snapshot"
33
command: .buildkite/scripts/cuvs-snapshot/update-current-snapshot-version.sh
44
agents:
5-
provider: gcp
6-
image: family/elasticsearch-ubuntu-2404-nvidia
7-
machineType: g2-standard-16
8-
zones: us-central1-b,us-central1-c
9-
diskSizeGb: 150
5+
provider: aws
6+
imagePrefix: elasticsearch-aws-ubuntu-2404-nvidia
7+
instanceType: g6.2xlarge
8+
diskSizeGb: 350
9+
diskType: gp3
10+
diskName: /dev/sda1
1011
- wait: ~
1112
- trigger: "elasticsearch-cuvs-run-tests"
1213
build:

‎.buildkite/pipelines/intake.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ steps:
6565
timeout_in_minutes: 300
6666
matrix:
6767
setup:
68-
BWC_VERSION: ["8.19.8", "9.1.8", "9.2.2", "9.3.0"]
68+
BWC_VERSION: ["8.19.9", "9.1.9", "9.2.3", "9.3.0"]
6969
agents:
7070
provider: gcp
7171
image: family/elasticsearch-ubuntu-2404

0 commit comments

Comments
 (0)