diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index 3c166410..145f6f0e 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -6,7 +6,7 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -54,14 +54,16 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque structure = { "agents": { "provider": "gcp", - "machineType": "n1-standard-4", - "image": "family/core-ubuntu-2204" + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204" }, "steps": []} steps = [] response = call_url_with_retry(RELEASES_URL) - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) # there are situations to manually run CIs with PR change, # set MANUAL_TARGET_BRANCH with upstream target branch and run @@ -69,21 +71,20 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH") print(f"Running with target_branch: {target_branch}") if target_branch == '8.x': - full_stack_version: typing.final = versions_json["snapshots"]["8.future"] + full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"] steps += generate_unit_and_integration_test_steps(full_stack_version, "true") elif target_branch == 'main': - full_stack_version: typing.final = versions_json["snapshots"][target_branch] + full_stack_version: typing.final = versions_yaml["snapshots"][target_branch] steps += generate_unit_and_integration_test_steps(full_stack_version, "true") else: # generate steps for the version if released - releases = versions_json["releases"] + releases = versions_yaml["releases"] for release_version in releases: if releases[release_version].startswith(target_branch): steps += generate_unit_and_integration_test_steps(releases[release_version], "false") break - # steps for snapshot version - snapshots = versions_json["snapshots"] + snapshots = versions_yaml["snapshots"] for snapshot_version in snapshots: if snapshots[snapshot_version].startswith(target_branch): steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false") diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py index 33fe1f80..57d2b314 100644 --- a/.buildkite/scripts/e2e-pipeline/generate-steps.py +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -6,7 +6,7 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh" @@ -45,7 +45,8 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: steps = [] response = call_url_with_retry(RELEASES_URL) - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) # there are situations to manually run CIs with PR change, # set MANUAL_TARGET_BRANCH with upstream target branch and run @@ -53,21 +54,20 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH") print(f"Running with target_branch: {target_branch}") if target_branch == '8.x': - full_stack_version: typing.final = versions_json["snapshots"]["8.future"] + full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"] steps.append(generate_test_step(full_stack_version, target_branch, "true")) elif target_branch == 'main': - full_stack_version: typing.final = versions_json["snapshots"][target_branch] + full_stack_version: typing.final = versions_yaml["snapshots"][target_branch] steps.append(generate_test_step(full_stack_version, target_branch, "true")) else: # generate steps for the version if released - releases = versions_json["releases"] + releases = versions_yaml["releases"] for release_version in releases: if releases[release_version].startswith(target_branch): steps.append(generate_test_step(releases[release_version], target_branch, "false")) break - # steps for snapshot version - snapshots = versions_json["snapshots"] + snapshots = versions_yaml["snapshots"] for snapshot_version in snapshots: if snapshots[snapshot_version].startswith(target_branch): steps.append(generate_test_step(snapshots[snapshot_version], target_branch, "false")) diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index de431089..f29d9384 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -6,8 +6,9 @@ from ruamel.yaml import YAML -RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" -TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml" +RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml" +TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \ + "-request-test-matrix.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -60,14 +61,16 @@ def make_matrix_version_key(branch: str) -> str: structure = { "agents": { "provider": "gcp", - "machineType": "n1-standard-4", - "image": "family/core-ubuntu-2204" + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204" }, "steps": []} steps = [] response = call_url_with_retry(RELEASES_URL) - versions_json = response.json() + yaml = YAML(typ='safe') + versions_yaml: typing.final = yaml.load(response.text) matrix_map = call_url_with_retry(TEST_MATRIX_URL) matrix_map_yaml = YAML().load(matrix_map.text) @@ -87,13 +90,13 @@ def make_matrix_version_key(branch: str) -> str: print(f"matrix_releases: {matrix_releases}") print(f"matrix_snapshots: {matrix_snapshots}") for matrix_release in matrix_releases: - full_stack_version: typing.final = versions_json["releases"].get(matrix_release) + full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release) # noop, if they are declared in the matrix but not in the release if full_stack_version is not None: steps += generate_unit_and_integration_test_steps(full_stack_version, "false") for matrix_snapshot in matrix_snapshots: - full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot) + full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot) # noop, if they are declared in the matrix but not in the snapshot if full_stack_version is not None: steps += generate_unit_and_integration_test_steps(full_stack_version, "true") diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index b462caf4..3208a200 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1 +1,14 @@ -mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh \ No newline at end of file +#!/usr/bin/env bash + +export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21" +export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH" +eval "$(rbenv init -)" + +if [ -z "$TARGET_BRANCH" ]; then + echo "Target branch is not specified, using default branch: main or BK defined" +else + echo "Changing the branch for ${TARGET_BRANCH}" + git checkout "$TARGET_BRANCH" +fi + +mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh