[8.19] (backport #358) Update buildkite script to look for new logstash releases location (#361)

* Update buildkite script to look for new logstash releases location (#358)

(cherry picked from commit febb207)

# Conflicts:
#	.buildkite/scripts/build-pipeline/generate-steps.py
#	.buildkite/scripts/e2e-pipeline/generate-steps.py
#	.buildkite/scripts/pull-request-pipeline/generate-steps.py
#	.buildkite/scripts/run_tests.sh

* fix merge conflicts

---------

Co-authored-by: Cas Donoghue <[email protected]>

Author: mergify[bot]

.buildkite/scripts/build-pipeline/generate-steps.py

@@ -6,7 +6,7 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
 
 
@@ -54,36 +54,37 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n1-standard-4",
-            "image": "family/core-ubuntu-2204"
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
         },
         "steps": []}
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
-    versions_json = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     # there are situations to manually run CIs with PR change,
     # set MANUAL_TARGET_BRANCH with upstream target branch and run
     manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH")
     target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH")
     print(f"Running with target_branch: {target_branch}")
     if target_branch == '8.x':
-        full_stack_version: typing.final = versions_json["snapshots"]["8.future"]
+        full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"]
         steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
     elif target_branch == 'main':
-        full_stack_version: typing.final = versions_json["snapshots"][target_branch]
+        full_stack_version: typing.final = versions_yaml["snapshots"][target_branch]
         steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
     else:
         # generate steps for the version if released
-        releases = versions_json["releases"]
+        releases = versions_yaml["releases"]
         for release_version in releases:
             if releases[release_version].startswith(target_branch):
                 steps += generate_unit_and_integration_test_steps(releases[release_version], "false")
                 break
-
         # steps for snapshot version
-        snapshots = versions_json["snapshots"]
+        snapshots = versions_yaml["snapshots"]
         for snapshot_version in snapshots:
             if snapshots[snapshot_version].startswith(target_branch):
                 steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false")

.buildkite/scripts/e2e-pipeline/generate-steps.py

@@ -6,7 +6,7 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"
 
 
@@ -45,29 +45,29 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
-    versions_json = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     # there are situations to manually run CIs with PR change,
     # set MANUAL_TARGET_BRANCH with upstream target branch and run
     manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH")
     target_branch: typing.final = manually_set_target_branch if manually_set_target_branch else os.getenv("TARGET_BRANCH")
     print(f"Running with target_branch: {target_branch}")
     if target_branch == '8.x':
-        full_stack_version: typing.final = versions_json["snapshots"]["8.future"]
+        full_stack_version: typing.final = versions_yaml["snapshots"]["8.future"]
         steps.append(generate_test_step(full_stack_version, target_branch, "true"))
     elif target_branch == 'main':
-        full_stack_version: typing.final = versions_json["snapshots"][target_branch]
+        full_stack_version: typing.final = versions_yaml["snapshots"][target_branch]
         steps.append(generate_test_step(full_stack_version, target_branch, "true"))
     else:
         # generate steps for the version if released
-        releases = versions_json["releases"]
+        releases = versions_yaml["releases"]
         for release_version in releases:
             if releases[release_version].startswith(target_branch):
                 steps.append(generate_test_step(releases[release_version], target_branch, "false"))
                 break
-
         # steps for snapshot version
-        snapshots = versions_json["snapshots"]
+        snapshots = versions_yaml["snapshots"]
         for snapshot_version in snapshots:
             if snapshots[snapshot_version].startswith(target_branch):
                 steps.append(generate_test_step(snapshots[snapshot_version], target_branch, "false"))

.buildkite/scripts/pull-request-pipeline/generate-steps.py

@@ -6,8 +6,9 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
-TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
+TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \
+                  "-request-test-matrix.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
 
 
@@ -60,14 +61,16 @@ def make_matrix_version_key(branch: str) -> str:
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n1-standard-4",
-            "image": "family/core-ubuntu-2204"
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
         },
         "steps": []}
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
-    versions_json = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     matrix_map = call_url_with_retry(TEST_MATRIX_URL)
     matrix_map_yaml = YAML().load(matrix_map.text)
@@ -87,13 +90,13 @@ def make_matrix_version_key(branch: str) -> str:
     print(f"matrix_releases: {matrix_releases}")
     print(f"matrix_snapshots: {matrix_snapshots}")
     for matrix_release in matrix_releases:
-        full_stack_version: typing.final = versions_json["releases"].get(matrix_release)
+        full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release)
         # noop, if they are declared in the matrix but not in the release
         if full_stack_version is not None:
             steps += generate_unit_and_integration_test_steps(full_stack_version, "false")
 
     for matrix_snapshot in matrix_snapshots:
-        full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot)
+        full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot)
         # noop, if they are declared in the matrix but not in the snapshot
         if full_stack_version is not None:
             steps += generate_unit_and_integration_test_steps(full_stack_version, "true")

.buildkite/scripts/run_tests.sh

@@ -1 +1,14 @@
-mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
+#!/usr/bin/env bash
+
+export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21"
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH"
+eval "$(rbenv init -)"
+
+if [ -z "$TARGET_BRANCH" ]; then
+  echo "Target branch is not specified, using default branch: main or BK defined"
+else
+  echo "Changing the branch for ${TARGET_BRANCH}"
+  git checkout "$TARGET_BRANCH"
+fi
+
+mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh