Update buildkite script to look for new logstash releases location (#358)

donoghuc · mergify[bot] · commit fc932992ef08 · 2025-08-07T20:41:18.000Z
(cherry picked from commit febb207) # Conflicts: # .buildkite/scripts/build-pipeline/generate-steps.py # .buildkite/scripts/e2e-pipeline/generate-steps.py # .buildkite/scripts/pull-request-pipeline/generate-steps.py # .buildkite/scripts/run_tests.sh
diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py
@@ -6,7 +6,7 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
 
 
@@ -54,13 +54,15 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n1-standard-4",
-            "image": "family/core-ubuntu-2204"
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
         },
         "steps": []}
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
+<<<<<<< HEAD
     versions_json = response.json()
 
     # there are situations to manually run CIs with PR change,
@@ -81,6 +83,21 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
             if releases[release_version].startswith(target_branch):
                 steps += generate_unit_and_integration_test_steps(releases[release_version], "false")
                 break
+=======
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
+
+    # Use BUILDKITE_SOURCE to figure out PR merge or schedule.
+    # If PR merge, no need to run builds on all branches, target branch will be good
+    #   - webhook when PR gets merged
+    #   - schedule when daily schedule starts
+    #   - ui when manually kicking job from BK UI
+    #       - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
+    bk_source = os.getenv("BUILDKITE_SOURCE")
+    bk_branch = os.getenv("BUILDKITE_BRANCH")
+    steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
+        else generate_steps_for_main_branch(versions_yaml)
+>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
 
         # steps for snapshot version
         snapshots = versions_json["snapshots"]
diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py
@@ -6,7 +6,7 @@
 
 from ruamel.yaml import YAML
 
-RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"
 
 
@@ -45,6 +45,7 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
+<<<<<<< HEAD
     versions_json = response.json()
 
     # there are situations to manually run CIs with PR change,
@@ -65,6 +66,21 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
             if releases[release_version].startswith(target_branch):
                 steps.append(generate_test_step(releases[release_version], target_branch, "false"))
                 break
+=======
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
+
+    # Use BUILDKITE_SOURCE to figure out PR merge or schedule.
+    # If PR merge, no need to run builds on all branches, target branch will be good
+    #   - webhook when PR gets merged
+    #   - schedule when daily schedule starts
+    #   - ui when manually kicking job from BK UI
+    #       - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
+    bk_source = os.getenv("BUILDKITE_SOURCE")
+    bk_branch = os.getenv("BUILDKITE_BRANCH")
+    steps = generate_steps_for_scheduler(versions_yaml) if (bk_source == "schedule" or bk_branch == "main") \
+        else generate_steps_for_main_branch(versions_yaml)
+>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
 
         # steps for snapshot version
         snapshots = versions_json["snapshots"]
diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py
@@ -6,8 +6,14 @@
 
 from ruamel.yaml import YAML
 
+<<<<<<< HEAD
 RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
 TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull-request-test-matrix.yml"
+=======
+RELEASES_URL = "https://raw.githubusercontent.com/logstash-plugins/.ci/refs/heads/1.x/logstash-versions.yml"
+TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \
+                  "-request-test-matrix.yml"
+>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
 TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
 
 
@@ -60,14 +66,16 @@ def make_matrix_version_key(branch: str) -> str:
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n1-standard-4",
-            "image": "family/core-ubuntu-2204"
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204"
         },
         "steps": []}
 
     steps = []
     response = call_url_with_retry(RELEASES_URL)
-    versions_json = response.json()
+    yaml = YAML(typ='safe')
+    versions_yaml: typing.final = yaml.load(response.text)
 
     matrix_map = call_url_with_retry(TEST_MATRIX_URL)
     matrix_map_yaml = YAML().load(matrix_map.text)
@@ -87,13 +95,13 @@ def make_matrix_version_key(branch: str) -> str:
     print(f"matrix_releases: {matrix_releases}")
     print(f"matrix_snapshots: {matrix_snapshots}")
     for matrix_release in matrix_releases:
-        full_stack_version: typing.final = versions_json["releases"].get(matrix_release)
+        full_stack_version: typing.final = versions_yaml["releases"].get(matrix_release)
         # noop, if they are declared in the matrix but not in the release
         if full_stack_version is not None:
             steps += generate_unit_and_integration_test_steps(full_stack_version, "false")
 
     for matrix_snapshot in matrix_snapshots:
-        full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot)
+        full_stack_version: typing.final = versions_yaml["snapshots"].get(matrix_snapshot)
         # noop, if they are declared in the matrix but not in the snapshot
         if full_stack_version is not None:
             steps += generate_unit_and_integration_test_steps(full_stack_version, "true")
diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh
@@ -1 +1,18 @@
-mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
+<<<<<<< HEAD
+mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh
+=======
+#!/usr/bin/env bash
+
+export JAVA_HOME="/opt/buildkite-agent/.java/adoptiumjdk_21"
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$JAVA_HOME:$PATH"
+eval "$(rbenv init -)"
+
+if [ -z "$TARGET_BRANCH" ]; then
+  echo "Target branch is not specified, using default branch: main or BK defined"
+else
+  echo "Changing the branch for ${TARGET_BRANCH}"
+  git checkout "$TARGET_BRANCH"
+fi
+
+mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' '*logstash-versions*' && .ci/docker-setup.sh && .ci/docker-run.sh
+>>>>>>> febb207 (Update buildkite script to look for new logstash releases location (#358))
