Skip to content

Update buildkite script to look for new logstash releases location #358

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 7, 2025
Merged

Update buildkite script to look for new logstash releases location #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c5987ae
Update buildkite script to look for new logstash releases location
donoghuc Aug 6, 2025
ef0ce13
Use the same vm for all tests
donoghuc Aug 7, 2025
19bd809
set java home
donoghuc Aug 7, 2025
03643ad
Use default disk size on test runners
donoghuc Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions .buildkite/scripts/build-pipeline/generate-steps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from ruamel.yaml import YAML

RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"


Expand Down Expand Up @@ -63,13 +63,15 @@ def generate_steps_for_main_branch(versions) -> list:
structure = {
"agents": {
"provider": "gcp",
"machineType": "n1-standard-4",
"image": "family/core-ubuntu-2204"
"machineType": "n2-standard-4",
"imageProject": "elastic-images-prod",
"image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
},
"steps": []}

response = call_url_with_retry(RELEASES_URL)
versions_json: typing.final = response.json()
yaml = YAML(typ='safe')
versions_yaml: typing.final = yaml.load(response.text)

# Use BUILDKITE_SOURCE to figure out PR merge or schedule.
# If PR merge, no need to run builds on all branches, target branch will be good
Expand All @@ -79,8 +81,8 @@ def generate_steps_for_main_branch(versions) -> list:
# - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
bk_source = os.getenv("BUILDKITE_SOURCE")
bk_branch = os.getenv("BUILDKITE_BRANCH")
steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \
else generate_steps_for_main_branch(versions_json)
steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
else generate_steps_for_main_branch(versions_yaml)

group_desc = f"Build steps"
key_desc = "build-steps"
Expand Down
9 changes: 5 additions & 4 deletions .buildkite/scripts/e2e-pipeline/generate-steps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from requests.adapters import HTTPAdapter, Retry
from ruamel.yaml import YAML

RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"


Expand Down Expand Up @@ -67,7 +67,8 @@ def generate_steps_for_main_branch(versions) -> list:
"steps": []}

response = call_url_with_retry(RELEASES_URL)
versions_json: typing.final = response.json()
yaml = YAML(typ='safe')
versions_yaml: typing.final = yaml.load(response.text)

# Use BUILDKITE_SOURCE to figure out PR merge or schedule.
# If PR merge, no need to run builds on all branches, target branch will be good
Expand All @@ -77,8 +78,8 @@ def generate_steps_for_main_branch(versions) -> list:
# - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
bk_source = os.getenv("BUILDKITE_SOURCE")
bk_branch = os.getenv("BUILDKITE_BRANCH")
steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \
else generate_steps_for_main_branch(versions_json)
steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
else generate_steps_for_main_branch(versions_yaml)

group_desc = f"E2E steps"
key_desc = "e2e-steps"
Expand Down
14 changes: 8 additions & 6 deletions .buildkite/scripts/pull-request-pipeline/generate-steps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from ruamel.yaml import YAML

RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \
"-request-test-matrix.yml"
TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
Expand Down Expand Up @@ -61,14 +61,16 @@ def make_matrix_version_key(branch: str) -> str:
structure = {
"agents": {
"provider": "gcp",
"machineType": "n1-standard-4",
"image": "family/core-ubuntu-2204"
"machineType": "n2-standard-4",
"imageProject": "elastic-images-prod",
"image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
},
"steps": []}

steps = []
response = call_url_with_retry(RELEASES_URL)
versions_json = response.json()
yaml = YAML(typ='safe')
versions_yaml: typing.final = yaml.load(response.text)

matrix_map = call_url_with_retry(TEST_MATRIX_URL)
matrix_map_yaml = YAML().load(matrix_map.text)
Expand All @@ -88,13 +90,13 @@ def make_matrix_version_key(branch: str) -> str:
print(f"matrix_releases: {matrix_releases}")
print(f"matrix_snapshots: {matrix_snapshots}")
for matrix_release in matrix_releases:
full_stack_version: typing.final = versions_json["releases"].get(matrix_release)
full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release)
# noop, if they are declared in the matrix but not in the release
if full_stack_version is not None:
steps += generate_unit_and_integration_test_steps(full_stack_version, "false")

for matrix_snapshot in matrix_snapshots:
full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot)
full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot)
# noop, if they are declared in the matrix but not in the snapshot
if full_stack_version is not None:
steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
Expand Down
7 changes: 6 additions & 1 deletion .buildkite/scripts/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
#!/usr/bin/env bash

export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21"
export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH"
eval "$(rbenv init -)"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah right, this was a requirement!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, i will be re-visiting this as part of https://github.com/elastic/ingest-dev/issues/5827


if [ -z "$TARGET_BRANCH" ]; then
echo "Target branch is not specified, using default branch: main or BK defined"
Expand All @@ -6,4 +11,4 @@ else
git checkout "$TARGET_BRANCH"
fi

mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh