From c5987ae610abe847bdcb9760749e249be249a61d Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 6 Aug 2025 16:09:15 -0700 Subject: [PATCH 1/4] Update buildkite script to look for new logstash releases location This commit updates the buildkite scripts to look in the `.ci` repo in the logstash-plugins org for the new location of logstash releases. --- .buildkite/scripts/build-pipeline/generate-steps.py | 9 +++++---- .buildkite/scripts/e2e-pipeline/generate-steps.py | 9 +++++---- .../scripts/pull-request-pipeline/generate-steps.py | 9 +++++---- .buildkite/scripts/run_tests.sh | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index b032dce0..99083b5e 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -6,7 +6,7 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -69,7 +69,8 @@ def generate_steps_for_main_branch(versions) -> list: "steps": []} response = call_url_with_retry(RELEASES_URL) - versions_json: typing.final = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) # Use BUILDKITE_SOURCE to figure out PR merge or schedule. # If PR merge, no need to run builds on all branches, target branch will be good @@ -79,8 +80,8 @@ def generate_steps_for_main_branch(versions) -> list: # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH bk_source = os.getenv("BUILDKITE_SOURCE") bk_branch = os.getenv("BUILDKITE_BRANCH") - steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \ - else generate_steps_for_main_branch(versions_json) + steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_yaml) group_desc = f"Build steps" key_desc = "build-steps" diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py index 3a6c30bd..37d31bcf 100644 --- a/.buildkite/scripts/e2e-pipeline/generate-steps.py +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -5,7 +5,7 @@ from requests.adapters import HTTPAdapter, Retry from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh" @@ -67,7 +67,8 @@ def generate_steps_for_main_branch(versions) -> list: "steps": []} response = call_url_with_retry(RELEASES_URL) - versions_json: typing.final = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) # Use BUILDKITE_SOURCE to figure out PR merge or schedule. # If PR merge, no need to run builds on all branches, target branch will be good @@ -77,8 +78,8 @@ def generate_steps_for_main_branch(versions) -> list: # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH bk_source = os.getenv("BUILDKITE_SOURCE") bk_branch = os.getenv("BUILDKITE_BRANCH") - steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \ - else generate_steps_for_main_branch(versions_json) + steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_yaml) group_desc = f"E2E steps" key_desc = "e2e-steps" diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index 5a4c7cab..9bb62fab 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -6,7 +6,7 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \ "-request-test-matrix.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -68,7 +68,8 @@ def make_matrix_version_key(branch: str) -> str: steps = [] response = call_url_with_retry(RELEASES_URL) - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) matrix_map = call_url_with_retry(TEST_MATRIX_URL) matrix_map_yaml = YAML().load(matrix_map.text) @@ -88,13 +89,13 @@ def make_matrix_version_key(branch: str) -> str: print(f"matrix_releases: {matrix_releases}") print(f"matrix_snapshots: {matrix_snapshots}") for matrix_release in matrix_releases: - full_stack_version: typing.final = versions_json["releases"].get(matrix_release) + full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release) # noop, if they are declared in the matrix but not in the release if full_stack_version is not None: steps += generate_unit_and_integration_test_steps(full_stack_version, "false") for matrix_snapshot in matrix_snapshots: - full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot) + full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot) # noop, if they are declared in the matrix but not in the snapshot if full_stack_version is not None: steps += generate_unit_and_integration_test_steps(full_stack_version, "true") diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index 9b8b7257..e5303b3a 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -6,4 +6,4 @@ else git checkout "$TARGET_BRANCH" fi -mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh \ No newline at end of file +mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh \ No newline at end of file From ef0ce136385ca57a79be5e8fb0ee3f4a76ba0fda Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 7 Aug 2025 10:49:20 -0700 Subject: [PATCH 2/4] Use the same vm for all tests Use images we are building for LS testing for all BK pipelines. --- .buildkite/scripts/build-pipeline/generate-steps.py | 6 ++++-- .buildkite/scripts/pull-request-pipeline/generate-steps.py | 6 ++++-- .buildkite/scripts/run_tests.sh | 5 +++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index 99083b5e..832f737c 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -63,8 +63,10 @@ def generate_steps_for_main_branch(versions) -> list: structure = { "agents": { "provider": "gcp", - "machineType": "n1-standard-4", - "image": "family/core-ubuntu-2204" + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204", + "diskSizeGb": 120 }, "steps": []} diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index 9bb62fab..e3e9691b 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -61,8 +61,10 @@ def make_matrix_version_key(branch: str) -> str: structure = { "agents": { "provider": "gcp", - "machineType": "n1-standard-4", - "image": "family/core-ubuntu-2204" + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204", + "diskSizeGb": 120 }, "steps": []} diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index e5303b3a..a250acc5 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1,3 +1,8 @@ +#!/usr/bin/env bash + +export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$PATH" +export JAVA_HOME="/opt/buildkite-agent/.java" +eval "$(rbenv init -)" if [ -z "$TARGET_BRANCH" ]; then echo "Target branch is not specified, using default branch: main or BK defined" From 19bd809ab396351dd283734e1d852529aaf16b12 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 7 Aug 2025 11:41:22 -0700 Subject: [PATCH 3/4] set java home --- .buildkite/scripts/run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index a250acc5..2c7f42ff 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$PATH" -export JAVA_HOME="/opt/buildkite-agent/.java" +export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21" +export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH" eval "$(rbenv init -)" if [ -z "$TARGET_BRANCH" ]; then From 03643ad9765225fd70f2f69e43d370aaff5f4520 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 7 Aug 2025 12:25:12 -0700 Subject: [PATCH 4/4] Use default disk size on test runners --- .buildkite/scripts/build-pipeline/generate-steps.py | 3 +-- .buildkite/scripts/pull-request-pipeline/generate-steps.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index 832f737c..4310ccd0 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -65,8 +65,7 @@ def generate_steps_for_main_branch(versions) -> list: "provider": "gcp", "machineType": "n2-standard-4", "imageProject": "elastic-images-prod", - "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204", - "diskSizeGb": 120 + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204" }, "steps": []} diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index e3e9691b..f29d9384 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -63,8 +63,7 @@ def make_matrix_version_key(branch: str) -> str: "provider": "gcp", "machineType": "n2-standard-4", "imageProject": "elastic-images-prod", - "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204", - "diskSizeGb": 120 + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204" }, "steps": []}