Skip to content

Commit 12da8b3

Browse files
breskebybreskeby
committed
Rerun only tests failed in previous build job iteration
1 parent 537172e commit 12da8b3

File tree

13 files changed

+357
-217
lines changed

13 files changed

+357
-217
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@ export GRADLE_BUILD_CACHE_USERNAME
4444
GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
4545
export GRADLE_BUILD_CACHE_PASSWORD
4646

47-
DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)"
47+
DEVELOCITY_API_ACCESS_KEY=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
48+
export DEVELOCITY_API_ACCESS_KEY
49+
50+
DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$DEVELOCITY_API_ACCESS_KEY"
4851
export DEVELOCITY_ACCESS_KEY
4952

5053
BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
@@ -112,6 +115,7 @@ if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
112115
export PERF_METRICS_PASSWORD
113116
fi
114117

118+
115119
# Authenticate to the Docker Hub public read-only registry
116120
if which docker > /dev/null 2>&1; then
117121
DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"
@@ -141,6 +145,21 @@ EOF
141145
EOF
142146
fi
143147

148+
if [[ "${SMART_RETRIES:-}" == "true" && "${BUILDKITE_RETRY_COUNT:-0}" -gt 0 ]]; then
149+
echo "--- Resolving previously failed tests"
150+
BUILD_JSON=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -X GET "https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}")
151+
ORIGIN_JOB_ID=$(printf '%s\n' "$BUILD_JSON" | jq -r--arg jobId "$BUILDKITE_JOB_ID" ' .jobs[] | select(.id == $jobId) | .retry_source.job_id')
152+
BUILD_SCAN_URL=$(printf '%s\n' "$BUILD_JSON" | jq -r --arg job_id "$ORIGIN_JOB_ID" '.meta_data["build-scan-" + $job_id]')
153+
BUILD_SCAN_ID=$(echo "$BUILD_SCAN_URL" | sed 's|.*/s/||')
154+
DEVELOCITY_FAILED_TEST_API_URL="https://gradle-enterprise.elastic.co/api/tests/build/${BUILD_SCAN_ID}?testOutcomes=failed"
155+
156+
curl --request GET \
157+
--url $DEVELOCITY_FAILED_TEST_API_URL \
158+
--header 'accept: application/json' \
159+
--header "authorization: Bearer $DEVELOCITY_API_ACCESS_KEY" \
160+
--header 'content-type: application/json' | gunzip | jq '.' &> failed-tests.json
161+
fi
162+
144163
# Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2
145164
# We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way
146165
# And they fail. This sets up a manual entry for the hostname in dnsmasq.

‎.buildkite/pipelines/periodic.bwc.template.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717
- signal_reason: agent_stop
1818
limit: 3
1919
- exit_status: "1"
20-
limit: 2
20+
limit: 1

‎.buildkite/pipelines/periodic.template.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
env:
2+
SMART_RETRIES: "true"
13
steps:
24
- group: bwc
35
steps: $BWC_STEPS
@@ -132,7 +134,7 @@ steps:
132134
- signal_reason: agent_stop
133135
limit: 3
134136
- exit_status: "1"
135-
limit: 2
137+
limit: 1
136138
agents:
137139
provider: gcp
138140
image: family/elasticsearch-ubuntu-2404

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
# This file is auto-generated. See .buildkite/pipelines/periodic.template.yml
2+
env:
3+
SMART_RETRIES: "true"
24
steps:
35
- group: bwc
46
steps:
@@ -20,6 +22,8 @@ steps:
2022
signal_reason: none
2123
- signal_reason: agent_stop
2224
limit: 3
25+
- exit_status: "1"
26+
limit: 1
2327

2428
- label: 8.1.3 / bwc
2529
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.1.3#bwcTest
@@ -39,8 +43,8 @@ steps:
3943
signal_reason: none
4044
- signal_reason: agent_stop
4145
limit: 3
42-
- exit_status: "-1"
43-
limit: 2
46+
- exit_status: "1"
47+
limit: 1
4448

4549
- label: 8.2.3 / bwc
4650
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.2.3#bwcTest
@@ -60,8 +64,8 @@ steps:
6064
signal_reason: none
6165
- signal_reason: agent_stop
6266
limit: 3
63-
- exit_status: "-1"
64-
limit: 2
67+
- exit_status: "1"
68+
limit: 1
6569

6670
- label: 8.3.3 / bwc
6771
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.3.3#bwcTest
@@ -82,7 +86,7 @@ steps:
8286
- signal_reason: agent_stop
8387
limit: 3
8488
- exit_status: "1"
85-
limit: 2
89+
limit: 1
8690

8791
- label: 8.4.3 / bwc
8892
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.4.3#bwcTest
@@ -103,7 +107,7 @@ steps:
103107
- signal_reason: agent_stop
104108
limit: 3
105109
- exit_status: "1"
106-
limit: 2
110+
limit: 1
107111

108112
- label: 8.5.3 / bwc
109113
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.5.3#bwcTest
@@ -124,7 +128,7 @@ steps:
124128
- signal_reason: agent_stop
125129
limit: 3
126130
- exit_status: "1"
127-
limit: 2
131+
limit: 1
128132

129133
- label: 8.6.2 / bwc
130134
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.6.2#bwcTest
@@ -145,7 +149,7 @@ steps:
145149
- signal_reason: agent_stop
146150
limit: 3
147151
- exit_status: "1"
148-
limit: 2
152+
limit: 1
149153

150154
- label: 8.7.1 / bwc
151155
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.7.1#bwcTest
@@ -166,7 +170,7 @@ steps:
166170
- signal_reason: agent_stop
167171
limit: 3
168172
- exit_status: "1"
169-
limit: 2
173+
limit: 1
170174

171175
- label: 8.8.2 / bwc
172176
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.8.2#bwcTest
@@ -187,7 +191,7 @@ steps:
187191
- signal_reason: agent_stop
188192
limit: 3
189193
- exit_status: "1"
190-
limit: 2
194+
limit: 1
191195

192196
- label: 8.9.2 / bwc
193197
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.9.2#bwcTest
@@ -208,7 +212,7 @@ steps:
208212
- signal_reason: agent_stop
209213
limit: 3
210214
- exit_status: "1"
211-
limit: 2
215+
limit: 1
212216

213217
- label: 8.10.4 / bwc
214218
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest
@@ -229,7 +233,7 @@ steps:
229233
- signal_reason: agent_stop
230234
limit: 3
231235
- exit_status: "1"
232-
limit: 2
236+
limit: 1
233237

234238
- label: 8.11.4 / bwc
235239
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.4#bwcTest
@@ -250,7 +254,7 @@ steps:
250254
- signal_reason: agent_stop
251255
limit: 3
252256
- exit_status: "1"
253-
limit: 2
257+
limit: 1
254258

255259
- label: 8.12.2 / bwc
256260
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.2#bwcTest
@@ -271,7 +275,7 @@ steps:
271275
- signal_reason: agent_stop
272276
limit: 3
273277
- exit_status: "1"
274-
limit: 2
278+
limit: 1
275279

276280
- label: 8.13.4 / bwc
277281
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.4#bwcTest
@@ -292,7 +296,7 @@ steps:
292296
- signal_reason: agent_stop
293297
limit: 3
294298
- exit_status: "1"
295-
limit: 2
299+
limit: 1
296300

297301
- label: 8.14.3 / bwc
298302
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.3#bwcTest
@@ -313,7 +317,7 @@ steps:
313317
- signal_reason: agent_stop
314318
limit: 3
315319
- exit_status: "1"
316-
limit: 2
320+
limit: 1
317321

318322
- label: 8.15.5 / bwc
319323
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest
@@ -334,7 +338,7 @@ steps:
334338
- signal_reason: agent_stop
335339
limit: 3
336340
- exit_status: "1"
337-
limit: 2
341+
limit: 1
338342

339343
- label: 8.16.6 / bwc
340344
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
@@ -355,7 +359,7 @@ steps:
355359
- signal_reason: agent_stop
356360
limit: 3
357361
- exit_status: "1"
358-
limit: 2
362+
limit: 1
359363

360364
- label: 8.17.10 / bwc
361365
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.10#bwcTest
@@ -376,7 +380,7 @@ steps:
376380
- signal_reason: agent_stop
377381
limit: 3
378382
- exit_status: "1"
379-
limit: 2
383+
limit: 1
380384

381385
- label: 8.18.9 / bwc
382386
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.9#bwcTest
@@ -397,7 +401,7 @@ steps:
397401
- signal_reason: agent_stop
398402
limit: 3
399403
- exit_status: "1"
400-
limit: 2
404+
limit: 1
401405

402406
- label: 8.19.6 / bwc
403407
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.6#bwcTest
@@ -418,7 +422,7 @@ steps:
418422
- signal_reason: agent_stop
419423
limit: 3
420424
- exit_status: "1"
421-
limit: 2
425+
limit: 1
422426

423427
- label: 9.0.9 / bwc
424428
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.9#bwcTest
@@ -439,7 +443,7 @@ steps:
439443
- signal_reason: agent_stop
440444
limit: 3
441445
- exit_status: "1"
442-
limit: 2
446+
limit: 1
443447

444448
- label: 9.1.6 / bwc
445449
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.6#bwcTest
@@ -460,7 +464,7 @@ steps:
460464
- signal_reason: agent_stop
461465
limit: 3
462466
- exit_status: "1"
463-
limit: 2
467+
limit: 1
464468

465469
- label: 9.2.0 / bwc
466470
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.2.0#bwcTest
@@ -481,7 +485,7 @@ steps:
481485
- signal_reason: agent_stop
482486
limit: 3
483487
- exit_status: "1"
484-
limit: 2
488+
limit: 1
485489

486490
- label: 9.3.0 / bwc
487491
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.3.0#bwcTest
@@ -502,7 +506,7 @@ steps:
502506
- signal_reason: agent_stop
503507
limit: 3
504508
- exit_status: "1"
505-
limit: 2
509+
limit: 1
506510

507511
- label: concurrent-search-tests
508512
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
@@ -635,7 +639,7 @@ steps:
635639
- signal_reason: agent_stop
636640
limit: 3
637641
- exit_status: "1"
638-
limit: 2
642+
limit: 1
639643
agents:
640644
provider: gcp
641645
image: family/elasticsearch-ubuntu-2404

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
1717
import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener;
1818
import org.elasticsearch.gradle.internal.test.SimpleCommandLineArgumentProvider;
19-
import org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin;
19+
import org.elasticsearch.gradle.internal.test.rerun.RerunPlugin;
2020
import org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin;
2121
import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
2222
import org.gradle.api.Action;
@@ -65,7 +65,7 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
6565
public void apply(Project project) {
6666
project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
6767
var buildParams = loadBuildParams(project);
68-
project.getPluginManager().apply(TestRerunPlugin.class);
68+
project.getPluginManager().apply(RerunPlugin.class);
6969
project.getPluginManager().apply(GradleTestPolicySetupPlugin.class);
7070
// for fips mode check
7171
project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);

0 commit comments

Comments
 (0)