Skip to content

Commit 30d4bd5

Browse files
donoghucmergify[bot]
donoghuc
authored and
mergify[bot]
committed
Update buildkite script to look for new logstash releases location (#358)
(cherry picked from commit febb207) # Conflicts: # .buildkite/scripts/build-pipeline/generate-steps.py # .buildkite/scripts/e2e-pipeline/generate-steps.py # .buildkite/scripts/pull-request-pipeline/generate-steps.py # .buildkite/scripts/run_tests.sh
1 parent f1bcc11 commit 30d4bd5

File tree

4 files changed

+68
-10
lines changed

4 files changed

+68
-10
lines changed

‎.buildkite/scripts/build-pipeline/generate-steps.py

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from ruamel.yaml import YAML
88

9-
RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
9+
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
1010
TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
1111

1212

@@ -54,13 +54,15 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
5454
structure = {
5555
"agents": {
5656
"provider": "gcp",
57-
"machineType": "n1-standard-4",
58-
"image": "family/core-ubuntu-2204"
57+
"machineType": "n2-standard-4",
58+
"imageProject": "elastic-images-prod",
59+
"image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
5960
},
6061
"steps": []}
6162

6263
steps = []
6364
response = call_url_with_retry(RELEASES_URL)
65+
<<<<<<< HEAD
6466
versions_json = response.json()
6567

6668
# there are situations to manually run CIs with PR change,
@@ -81,6 +83,21 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
8183
if releases[release_version].startswith(target_branch):
8284
steps += generate_unit_and_integration_test_steps(releases[release_version], "false")
8385
break
86+
=======
87+
yaml = YAML(typ='safe')
88+
versions_yaml: typing.final = yaml.load(response.text)
89+
90+
# Use BUILDKITE_SOURCE to figure out PR merge or schedule.
91+
# If PR merge, no need to run builds on all branches, target branch will be good
92+
# - webhook when PR gets merged
93+
# - schedule when daily schedule starts
94+
# - ui when manually kicking job from BK UI
95+
# - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
96+
bk_source = os.getenv("BUILDKITE_SOURCE")
97+
bk_branch = os.getenv("BUILDKITE_BRANCH")
98+
steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
99+
else generate_steps_for_main_branch(versions_yaml)
100+
>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
84101

85102
# steps for snapshot version
86103
snapshots = versions_json["snapshots"]

‎.buildkite/scripts/e2e-pipeline/generate-steps.py

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from ruamel.yaml import YAML
88

9-
RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
9+
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
1010
TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"
1111

1212

@@ -45,6 +45,7 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
4545

4646
steps = []
4747
response = call_url_with_retry(RELEASES_URL)
48+
<<<<<<< HEAD
4849
versions_json = response.json()
4950

5051
# there are situations to manually run CIs with PR change,
@@ -65,6 +66,21 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
6566
if releases[release_version].startswith(target_branch):
6667
steps.append(generate_test_step(releases[release_version], target_branch, "false"))
6768
break
69+
=======
70+
yaml = YAML(typ='safe')
71+
versions_yaml: typing.final = yaml.load(response.text)
72+
73+
# Use BUILDKITE_SOURCE to figure out PR merge or schedule.
74+
# If PR merge, no need to run builds on all branches, target branch will be good
75+
# - webhook when PR gets merged
76+
# - schedule when daily schedule starts
77+
# - ui when manually kicking job from BK UI
78+
# - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
79+
bk_source = os.getenv("BUILDKITE_SOURCE")
80+
bk_branch = os.getenv("BUILDKITE_BRANCH")
81+
steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
82+
else generate_steps_for_main_branch(versions_yaml)
83+
>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
6884

6985
# steps for snapshot version
7086
snapshots = versions_json["snapshots"]

‎.buildkite/scripts/pull-request-pipeline/generate-steps.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,14 @@
66

77
from ruamel.yaml import YAML
88

9+
<<<<<<< HEAD
910
RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
1011
TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml"
12+
=======
13+
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
14+
TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \
15+
"-request-test-matrix.yml"
16+
>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
1117
TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
1218

1319

@@ -60,14 +66,16 @@ def make_matrix_version_key(branch: str) -> str:
6066
structure = {
6167
"agents": {
6268
"provider": "gcp",
63-
"machineType": "n1-standard-4",
64-
"image": "family/core-ubuntu-2204"
69+
"machineType": "n2-standard-4",
70+
"imageProject": "elastic-images-prod",
71+
"image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
6572
},
6673
"steps": []}
6774

6875
steps = []
6976
response = call_url_with_retry(RELEASES_URL)
70-
versions_json = response.json()
77+
yaml = YAML(typ='safe')
78+
versions_yaml: typing.final = yaml.load(response.text)
7179

7280
matrix_map = call_url_with_retry(TEST_MATRIX_URL)
7381
matrix_map_yaml = YAML().load(matrix_map.text)
@@ -87,13 +95,13 @@ def make_matrix_version_key(branch: str) -> str:
8795
print(f"matrix_releases: {matrix_releases}")
8896
print(f"matrix_snapshots: {matrix_snapshots}")
8997
for matrix_release in matrix_releases:
90-
full_stack_version: typing.final = versions_json["releases"].get(matrix_release)
98+
full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release)
9199
# noop, if they are declared in the matrix but not in the release
92100
if full_stack_version is not None:
93101
steps += generate_unit_and_integration_test_steps(full_stack_version, "false")
94102

95103
for matrix_snapshot in matrix_snapshots:
96-
full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot)
104+
full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot)
97105
# noop, if they are declared in the matrix but not in the snapshot
98106
if full_stack_version is not None:
99107
steps += generate_unit_and_integration_test_steps(full_stack_version, "true")

‎.buildkite/scripts/run_tests.sh

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,18 @@
1-
mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
1+
<<<<<<< HEAD
2+
mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
3+
=======
4+
#!/usr/bin/env bash
5+
6+
export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21"
7+
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH"
8+
eval "$(rbenv init -)"
9+
10+
if [ -z "$TARGET_BRANCH" ]; then
11+
echo "Target branch is not specified, using default branch: main or BK defined"
12+
else
13+
echo "Changing the branch for ${TARGET_BRANCH}"
14+
git checkout "$TARGET_BRANCH"
15+
fi
16+
17+
mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh
18+
>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))

0 commit comments

Comments
 (0)