Skip to content

Commit 9b0a20d

Browse files
mergify[bot]donoghuc
mergify[bot]
and
donoghuc
authored
[8.17] (backport #358) Update buildkite script to look for new logstash releases location (#359)
* Update buildkite script to look for new logstash releases location (#358) (cherry picked from commit febb207) # Conflicts: # .buildkite/scripts/build-pipeline/generate-steps.py # .buildkite/scripts/e2e-pipeline/generate-steps.py # .buildkite/scripts/pull-request-pipeline/generate-steps.py # .buildkite/scripts/run_tests.sh * fix merge conflict --------- Co-authored-by: Cas Donoghue <[email protected]>
1 parent a9eb739 commit 9b0a20d

File tree

4 files changed

+41
-24
lines changed

4 files changed

+41
-24
lines changed

‎.buildkite/scripts/build-pipeline/generate-steps.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from ruamel.yaml import YAML
88

9-
RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
9+
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
1010
TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
1111

1212

@@ -54,36 +54,37 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
5454
structure = {
5555
"agents": {
5656
"provider": "gcp",
57-
"machineType": "n1-standard-4",
58-
"image": "family/core-ubuntu-2204"
57+
"machineType": "n2-standard-4",
58+
"imageProject": "elastic-images-prod",
59+
"image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
5960
},
6061
"steps": []}
6162

6263
steps = []
6364
response = call_url_with_retry(RELEASES_URL)
64-
versions_json = response.json()
65+
yaml = YAML(typ='safe')
66+
versions_yaml: typing.final = yaml.load(response.text)
6567

6668
# there are situations to manually run CIs with PR change,
6769
# set MANUAL_TARGET_BRANCH with upstream target branch and run
6870
manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH")
6971
target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH")
7072
print(f"Running with target_branch: {target_branch}")
7173
if target_branch == '8.x':
72-
full_stack_version: typing.final = versions_json["snapshots"]["8.future"]
74+
full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"]
7375
steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
7476
elif target_branch == 'main':
75-
full_stack_version: typing.final = versions_json["snapshots"][target_branch]
77+
full_stack_version: typing.final = versions_yaml["snapshots"][target_branch]
7678
steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
7779
else:
7880
# generate steps for the version if released
79-
releases = versions_json["releases"]
81+
releases = versions_yaml["releases"]
8082
for release_version in releases:
8183
if releases[release_version].startswith(target_branch):
8284
steps += generate_unit_and_integration_test_steps(releases[release_version], "false")
8385
break
84-
8586
# steps for snapshot version
86-
snapshots = versions_json["snapshots"]
87+
snapshots = versions_yaml["snapshots"]
8788
for snapshot_version in snapshots:
8889
if snapshots[snapshot_version].startswith(target_branch):
8990
steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false")

‎.buildkite/scripts/e2e-pipeline/generate-steps.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from ruamel.yaml import YAML
88

9-
RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
9+
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
1010
TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"
1111

1212

@@ -45,29 +45,29 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
4545

4646
steps = []
4747
response = call_url_with_retry(RELEASES_URL)
48-
versions_json = response.json()
48+
yaml = YAML(typ='safe')
49+
versions_yaml: typing.final = yaml.load(response.text)
4950

5051
# there are situations to manually run CIs with PR change,
5152
# set MANUAL_TARGET_BRANCH with upstream target branch and run
5253
manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH")
5354
target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH")
5455
print(f"Running with target_branch: {target_branch}")
5556
if target_branch == '8.x':
56-
full_stack_version: typing.final = versions_json["snapshots"]["8.future"]
57+
full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"]
5758
steps.append(generate_test_step(full_stack_version, target_branch, "true"))
5859
elif target_branch == 'main':
59-
full_stack_version: typing.final = versions_json["snapshots"][target_branch]
60+
full_stack_version: typing.final = versions_yaml["snapshots"][target_branch]
6061
steps.append(generate_test_step(full_stack_version, target_branch, "true"))
6162
else:
6263
# generate steps for the version if released
63-
releases = versions_json["releases"]
64+
releases = versions_yaml["releases"]
6465
for release_version in releases:
6566
if releases[release_version].startswith(target_branch):
6667
steps.append(generate_test_step(releases[release_version], target_branch, "false"))
6768
break
68-
6969
# steps for snapshot version
70-
snapshots = versions_json["snapshots"]
70+
snapshots = versions_yaml["snapshots"]
7171
for snapshot_version in snapshots:
7272
if snapshots[snapshot_version].startswith(target_branch):
7373
steps.append(generate_test_step(snapshots[snapshot_version], target_branch, "false"))

‎.buildkite/scripts/pull-request-pipeline/generate-steps.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66

77
from ruamel.yaml import YAML
88

9-
RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
10-
TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml"
9+
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
10+
TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \
11+
"-request-test-matrix.yml"
1112
TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
1213

1314

@@ -60,14 +61,16 @@ def make_matrix_version_key(branch: str) -> str:
6061
structure = {
6162
"agents": {
6263
"provider": "gcp",
63-
"machineType": "n1-standard-4",
64-
"image": "family/core-ubuntu-2204"
64+
"machineType": "n2-standard-4",
65+
"imageProject": "elastic-images-prod",
66+
"image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
6567
},
6668
"steps": []}
6769

6870
steps = []
6971
response = call_url_with_retry(RELEASES_URL)
70-
versions_json = response.json()
72+
yaml = YAML(typ='safe')
73+
versions_yaml: typing.final = yaml.load(response.text)
7174

7275
matrix_map = call_url_with_retry(TEST_MATRIX_URL)
7376
matrix_map_yaml = YAML().load(matrix_map.text)
@@ -87,13 +90,13 @@ def make_matrix_version_key(branch: str) -> str:
8790
print(f"matrix_releases: {matrix_releases}")
8891
print(f"matrix_snapshots: {matrix_snapshots}")
8992
for matrix_release in matrix_releases:
90-
full_stack_version: typing.final = versions_json["releases"].get(matrix_release)
93+
full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release)
9194
# noop, if they are declared in the matrix but not in the release
9295
if full_stack_version is not None:
9396
steps += generate_unit_and_integration_test_steps(full_stack_version, "false")
9497

9598
for matrix_snapshot in matrix_snapshots:
96-
full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot)
99+
full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot)
97100
# noop, if they are declared in the matrix but not in the snapshot
98101
if full_stack_version is not None:
99102
steps += generate_unit_and_integration_test_steps(full_stack_version, "true")

‎.buildkite/scripts/run_tests.sh‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,14 @@
1-
mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
1+
#!/usr/bin/env bash
2+
3+
export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21"
4+
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH"
5+
eval "$(rbenv init -)"
6+
7+
if [ -z "$TARGET_BRANCH" ]; then
8+
echo "Target branch is not specified, using default branch: main or BK defined"
9+
else
10+
echo "Changing the branch for ${TARGET_BRANCH}"
11+
git checkout "$TARGET_BRANCH"
12+
fi
13+
14+
mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh

0 commit comments

Comments
 (0)