Update buildkite script to look for new logstash releases location (elastic#358)

Author: donoghuc

.buildkite/scripts/build-pipeline/generate-steps.py

@@ -6,7 +6,7 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
 
 
@@ -63,13 +63,15 @@ def generate_steps_for_main_branch(versions) -> list:
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n1-standard-4",
-            "image": "family/core-ubuntu-2204"
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
         },
         "steps": []}
 
     response = call_url_with_retry(RELEASES_URL)
-    versions_json: typing.final = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     # Use BUILDKITE_SOURCE to figure out PR merge or schedule.
     # If PR merge, no need to run builds on all branches, target branch will be good
@@ -79,8 +81,8 @@ def generate_steps_for_main_branch(versions) -> list:
     #       - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
     bk_source = os.getenv("BUILDKITE_SOURCE")
     bk_branch = os.getenv("BUILDKITE_BRANCH")
-    steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \
-        else generate_steps_for_main_branch(versions_json)
+    steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
+        else generate_steps_for_main_branch(versions_yaml)
 
     group_desc = f"Build steps"
     key_desc = "build-steps"

.buildkite/scripts/e2e-pipeline/generate-steps.py

@@ -5,7 +5,7 @@
 from requests.adapters import HTTPAdapter, Retry
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"
 
 
@@ -67,7 +67,8 @@ def generate_steps_for_main_branch(versions) -> list:
         "steps": []}
 
     response = call_url_with_retry(RELEASES_URL)
-    versions_json: typing.final = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     # Use BUILDKITE_SOURCE to figure out PR merge or schedule.
     # If PR merge, no need to run builds on all branches, target branch will be good
@@ -77,8 +78,8 @@ def generate_steps_for_main_branch(versions) -> list:
     #       - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
     bk_source = os.getenv("BUILDKITE_SOURCE")
     bk_branch = os.getenv("BUILDKITE_BRANCH")
-    steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \
-        else generate_steps_for_main_branch(versions_json)
+    steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
+        else generate_steps_for_main_branch(versions_yaml)
 
     group_desc = f"E2E steps"
     key_desc = "e2e-steps"

.buildkite/scripts/pull-request-pipeline/generate-steps.py

@@ -6,7 +6,7 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \
                   "-request-test-matrix.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
@@ -61,14 +61,16 @@ def make_matrix_version_key(branch: str) -> str:
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n1-standard-4",
-            "image": "family/core-ubuntu-2204"
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
         },
         "steps": []}
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
-    versions_json = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     matrix_map = call_url_with_retry(TEST_MATRIX_URL)
     matrix_map_yaml = YAML().load(matrix_map.text)
@@ -88,13 +90,13 @@ def make_matrix_version_key(branch: str) -> str:
     print(f"matrix_releases: {matrix_releases}")
     print(f"matrix_snapshots: {matrix_snapshots}")
     for matrix_release in matrix_releases:
-        full_stack_version: typing.final = versions_json["releases"].get(matrix_release)
+        full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release)
         # noop, if they are declared in the matrix but not in the release
         if full_stack_version is not None:
             steps += generate_unit_and_integration_test_steps(full_stack_version, "false")
 
     for matrix_snapshot in matrix_snapshots:
-        full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot)
+        full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot)
         # noop, if they are declared in the matrix but not in the snapshot
         if full_stack_version is not None:
             steps += generate_unit_and_integration_test_steps(full_stack_version, "true")

.buildkite/scripts/run_tests.sh

@@ -1,3 +1,8 @@
+#!/usr/bin/env bash
+
+export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21"
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH"
+eval "$(rbenv init -)"
 
 if [ -z "$TARGET_BRANCH" ]; then
   echo "Target branch is not specified, using default branch: main or BK defined"
@@ -6,4 +11,4 @@ else
   git checkout "$TARGET_BRANCH"
 fi
 
-mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
+mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh