From 225a119b5e1632674bcbe1a32622f49d5232f34a Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 9 Dec 2024 22:03:31 -0800 Subject: [PATCH 1/4] Rebuilds pull-request, build and E2E pipelines for main branch. --- .buildkite/build-pipeline.yml | 69 +++++-------- .buildkite/e2e-pipeline.yml | 65 +++++------- .buildkite/pull-request-pipeline.yml | 57 +++++------ .../scripts/build-pipeline/generate-steps.py | 70 +++++++++++++ .../scripts/{e2e => e2e-pipeline}/README.md | 6 +- .../scripts/{e2e => e2e-pipeline}/__init__.py | 0 .../{e2e => e2e-pipeline}/bootstrap.py | 2 +- .../config/pipeline.conf | 0 .../config/serverless_pipeline.conf | 0 .../scripts/e2e-pipeline/generate-steps.py | 59 +++++++++++ .../{e2e => e2e-pipeline}/logstash_stats.py | 0 .../scripts/{e2e => e2e-pipeline}/main.py | 4 +- .../{e2e => e2e-pipeline}/plugin_test.py | 0 .../{e2e => e2e-pipeline}/requirements.txt | 0 .../scripts/{e2e => e2e-pipeline}/util.py | 0 .../pull-request-pipeline/generate-steps.py | 98 +++++++++++++++++++ .buildkite/scripts/resolve_es_treeish.sh | 21 ---- .buildkite/scripts/run_e2e_tests.sh | 27 +---- .buildkite/scripts/run_tests.sh | 6 -- gradle.properties | 2 +- 20 files changed, 318 insertions(+), 168 deletions(-) create mode 100644 .buildkite/scripts/build-pipeline/generate-steps.py rename .buildkite/scripts/{e2e => e2e-pipeline}/README.md (89%) rename .buildkite/scripts/{e2e => e2e-pipeline}/__init__.py (100%) rename .buildkite/scripts/{e2e => e2e-pipeline}/bootstrap.py (99%) rename .buildkite/scripts/{e2e => e2e-pipeline}/config/pipeline.conf (100%) rename .buildkite/scripts/{e2e => e2e-pipeline}/config/serverless_pipeline.conf (100%) create mode 100644 .buildkite/scripts/e2e-pipeline/generate-steps.py rename .buildkite/scripts/{e2e => e2e-pipeline}/logstash_stats.py (100%) rename .buildkite/scripts/{e2e => e2e-pipeline}/main.py (90%) rename .buildkite/scripts/{e2e => e2e-pipeline}/plugin_test.py (100%) rename .buildkite/scripts/{e2e => e2e-pipeline}/requirements.txt (100%) rename .buildkite/scripts/{e2e => e2e-pipeline}/util.py (100%) create mode 100644 .buildkite/scripts/pull-request-pipeline/generate-steps.py delete mode 100755 .buildkite/scripts/resolve_es_treeish.sh diff --git a/.buildkite/build-pipeline.yml b/.buildkite/build-pipeline.yml index dbf9ee0c..538dfb0b 100644 --- a/.buildkite/build-pipeline.yml +++ b/.buildkite/build-pipeline.yml @@ -1,43 +1,28 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +- label: "Build pipeline" + command: | + #!/usr/bin/env bash + set -eo pipefail -agents: - provider: "gcp" - machineType: "n1-standard-4" - image: family/core-ubuntu-2204 - -steps: - - label: ":hammer: Build plugin with LS & ES 8.x :elasticsearch:" - # Builds with LS and ES last 8.x released version - # Runs integration tests on 8.x released versions - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - SNAPSHOT: false - INTEGRATION: true - SECURE_INTEGRATION: true - TARGET_BRANCH: "8.x" - - - label: ":hammer: Build plugin with LS 8.x-SNAPSHOT & ES `main` branch :elasticsearch:" - # Builds with LS last 8.x released version and ES main - # Runs integration tests on 8.x released versions - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - ELASTICSEARCH_TREEISH: "main" - SNAPSHOT: true - INTEGRATION: true - SECURE_INTEGRATION: true - - - label: ":hammer: Build plugin with LS & ES `main` branch :elasticsearch:" - # Builds with LS last 8.x released version and ES main - # Runs integration tests on 8.x released versions - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: "main" - ELASTICSEARCH_TREEISH: "main" - SNAPSHOT: true - INTEGRATION: true - SECURE_INTEGRATION: true \ No newline at end of file + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + python3 -m pip install requests + curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod a+x /usr/bin/yq + + echo "--- Generating dynamic steps" + set +e + python3 .buildkite/scripts/build-pipeline/generate-steps.py > pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem with generating pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml | yq . + fi + + set -eo pipefail + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/e2e-pipeline.yml b/.buildkite/e2e-pipeline.yml index 85bd871e..6c2951fc 100644 --- a/.buildkite/e2e-pipeline.yml +++ b/.buildkite/e2e-pipeline.yml @@ -1,39 +1,28 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +- label: "E2E pipeline" + command: | + #!/usr/bin/env bash + set -eo pipefail -agents: - provider: gcp - imageProject: elastic-images-prod - image: family/platform-ingest-logstash-multi-jdk-ubuntu-2204 - machineType: "n2-standard-4" - diskSizeGb: 120 - -steps: - # ------------- Run E2E tests --------------------- - - label: ":test_tube: Run E2E tests with LS 8.x :rocket:" - # uses the LS main & plugin 8.x branch when building the plugin - # Runs integration tests against 8.x release version - command: - - .buildkite/scripts/run_e2e_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - TARGET_BRANCH: "8.x" - - - label: ":test_tube: Run E2E tests with LS 8.x-SNAPSHOT :rocket:" - # uses the LS & plugin main branch when building the plugin - # Runs integration tests against 8.x-SNAPSHOT version - command: - - .buildkite/scripts/run_e2e_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - SNAPSHOT: true - TARGET_BRANCH: "8.x" - - - label: ":test_tube: Run E2E tests with LS `main` :rocket:" - # uses the LS & plugin main branch when building the plugin - # Runs integration tests against snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json - command: - - .buildkite/scripts/run_e2e_tests.sh - env: - ELASTIC_STACK_VERSION: main - SNAPSHOT: true - ELASTICSEARCH_TREEISH: main \ No newline at end of file + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + python3 -mpip install -r .buildkite/scripts/e2e-pipeline/requirements.txt + curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod a+x /usr/bin/yq + + echo "--- Generating dynamic steps" + set +e + python3 .buildkite/scripts/e2e-pipeline/generate-steps.py > pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem with generating pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml | yq . + fi + + set -eo pipefail + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/pull-request-pipeline.yml b/.buildkite/pull-request-pipeline.yml index ca7396a1..658d00ac 100644 --- a/.buildkite/pull-request-pipeline.yml +++ b/.buildkite/pull-request-pipeline.yml @@ -1,32 +1,27 @@ -# define a GCP VM agent to support container management (by default agent doesn't support) -agents: - provider: "gcp" - machineType: "n1-standard-4" - image: family/core-ubuntu-2204 +- label: "Pull request pipeline" + command: | + #!/usr/bin/env bash + set -eo pipefail + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + python3 -m pip install requests + curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod a+x /usr/bin/yq -steps: - # ------------- Unit tests --------------------- - - label: ":hammer: Unit tests with LS & ES main :docker:" - # Builds the plugin (with current changes) against LS and ES main - # Runs unit tests on LS & ES main docker - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: main - ELASTICSEARCH_TREEISH: main - INTEGRATION: false - SNAPSHOT: true - - # ------------- Integration tests --------------------- - - label: ":hammer: Integration tests with LS & ES main :docker:" - # Builds the plugin (with current changes) against LS and ES main - # Runs integration tests on snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: main - ELASTICSEARCH_TREEISH: main - SNAPSHOT: true - INTEGRATION: true - SECURE_INTEGRATION: true - LOG_LEVEL: "info" \ No newline at end of file + echo "--- Generating steps dynamically" + set +e + python3 .buildkite/scripts/pull-request-pipeline/generate-steps.py > pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem with generating pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml | yq . + fi + + set -eo pipefail + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py new file mode 100644 index 00000000..7e7bbc14 --- /dev/null +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -0,0 +1,70 @@ +import requests +import sys +import typing +from requests.adapters import HTTPAdapter, Retry + +from ruamel.yaml import YAML + +RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" + + +def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response: + schema = "https://" if "https://" in url else "http://" + session = requests.Session() + # retry on most common failures such as connection timeout(408), etc... + retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504]) + session.mount(schema, HTTPAdapter(max_retries=retries)) + return session.get(url) + + +def generate_test_step(stack_version, branch, snapshot) -> dict: + label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}" + return { + "label": label_integration_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "ELASTICSEARCH_TREEISH": branch, + "TARGET_BRANCH": branch, + "INTEGRATION": "true", + "SECURE_INTEGRATION": "true", + "LOG_LEVEL": "info" + } + } + + +if __name__ == "__main__": + structure = { + "agents": { + "provider": "gcp", + "machineType": "n1-standard-4", + "image": "family/core-ubuntu-2204" + }, + "steps": []} + + steps = [] + response = call_url_with_retry(RELEASES_URL) + versions_json = response.json() + snapshots = versions_json["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith("7.") or snapshots[snapshot_version].startswith("8.15"): + continue + full_stack_version = snapshots[snapshot_version] + version_parts = snapshots[snapshot_version].split(".") + major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" + branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions + steps.append(generate_test_step(full_stack_version, branch, "true")) + + group_desc = f"Build steps" + key_desc = "build-steps" + structure["steps"].append({ + "group": group_desc, + "key": key_desc, + "steps": steps, + }) + + print( + '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json') + YAML().dump(structure, sys.stdout) diff --git a/.buildkite/scripts/e2e/README.md b/.buildkite/scripts/e2e-pipeline/README.md similarity index 89% rename from .buildkite/scripts/e2e/README.md rename to .buildkite/scripts/e2e-pipeline/README.md index 70ca9d9c..f62be8fd 100644 --- a/.buildkite/scripts/e2e/README.md +++ b/.buildkite/scripts/e2e-pipeline/README.md @@ -32,19 +32,19 @@ In order to run tests with serverless, you also need to export `EC_API_KEY` whic In the pipelines, this will be automatically retrieved from Vault services. #### Stack version -E2E also requires `STACK_VERSION` (ex: "8.12.0") environment variable in order to test against. +E2E also requires `ELASTIC_STACK_VERSION` (ex: "8.12.0") environment variable in order to test against. Make sure to export it before running. In the Buildkite pipeline, this var will be resolved and exported. #### Installing dependencies Make sure you have python installed on you local ```bash -pip install -r .buildkite/scripts/e2e/requirements.txt +pip install -r .buildkite/scripts/e2e-pipeline/requirements.txt ``` ### Run Run the following command from the repo dir: ```bash -python3 .buildkite/scripts/e2e/main.py +python3 .buildkite/scripts/e2e-pipeline/main.py ``` ## Troubleshooting diff --git a/.buildkite/scripts/e2e/__init__.py b/.buildkite/scripts/e2e-pipeline/__init__.py similarity index 100% rename from .buildkite/scripts/e2e/__init__.py rename to .buildkite/scripts/e2e-pipeline/__init__.py diff --git a/.buildkite/scripts/e2e/bootstrap.py b/.buildkite/scripts/e2e-pipeline/bootstrap.py similarity index 99% rename from .buildkite/scripts/e2e/bootstrap.py rename to .buildkite/scripts/e2e-pipeline/bootstrap.py index 17bb8f9b..b70110b5 100644 --- a/.buildkite/scripts/e2e/bootstrap.py +++ b/.buildkite/scripts/e2e-pipeline/bootstrap.py @@ -122,7 +122,7 @@ def __reload_container(self) -> None: time.sleep(20) # give a time Logstash pipeline to fully start def __update_pipeline_config(self) -> None: - local_config_file_path = ".buildkite/scripts/e2e/config/" + local_config_file_path = ".buildkite/scripts/e2e-pipeline/config/" config_file = "serverless_pipeline.conf" if self.project_type == "serverless" else "pipeline.conf" local_config_file = local_config_file_path + config_file container_config_file_path = "/usr/share/logstash/pipeline/logstash.conf" diff --git a/.buildkite/scripts/e2e/config/pipeline.conf b/.buildkite/scripts/e2e-pipeline/config/pipeline.conf similarity index 100% rename from .buildkite/scripts/e2e/config/pipeline.conf rename to .buildkite/scripts/e2e-pipeline/config/pipeline.conf diff --git a/.buildkite/scripts/e2e/config/serverless_pipeline.conf b/.buildkite/scripts/e2e-pipeline/config/serverless_pipeline.conf similarity index 100% rename from .buildkite/scripts/e2e/config/serverless_pipeline.conf rename to .buildkite/scripts/e2e-pipeline/config/serverless_pipeline.conf diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py new file mode 100644 index 00000000..92b3986f --- /dev/null +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -0,0 +1,59 @@ +import sys +import typing +import util + +from ruamel.yaml import YAML + +RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh" + + +def generate_test_step(stack_version, es_treeish, snapshot) -> dict: + label_integration_test: typing.final = f"E2E tests for {stack_version}, snapshot: {snapshot}" + return { + "label": label_integration_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "ELASTICSEARCH_TREEISH": es_treeish, + "TARGET_BRANCH": branch + } + } + + +if __name__ == "__main__": + structure = { + "agents": { + "provider": "gcp", + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204", + "diskSizeGb": 120 + }, + "steps": []} + + steps = [] + response = util.call_url_with_retry(RELEASES_URL) + release_json = response.json() + snapshots = release_json["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith("7.") or snapshots[snapshot_version].startswith("8.15"): + continue + full_stack_version = snapshots[snapshot_version] + version_parts = snapshots[snapshot_version].split(".") + major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" + branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions + steps.append(generate_test_step(full_stack_version, branch, "true")) + + group_desc = f"E2E steps" + key_desc = "e2e-steps" + structure["steps"].append({ + "group": group_desc, + "key": key_desc, + "steps": steps, + }) + + print( + '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json') + YAML().dump(structure, sys.stdout) diff --git a/.buildkite/scripts/e2e/logstash_stats.py b/.buildkite/scripts/e2e-pipeline/logstash_stats.py similarity index 100% rename from .buildkite/scripts/e2e/logstash_stats.py rename to .buildkite/scripts/e2e-pipeline/logstash_stats.py diff --git a/.buildkite/scripts/e2e/main.py b/.buildkite/scripts/e2e-pipeline/main.py similarity index 90% rename from .buildkite/scripts/e2e/main.py rename to .buildkite/scripts/e2e-pipeline/main.py index 09e2c41e..de6d5ca6 100644 --- a/.buildkite/scripts/e2e/main.py +++ b/.buildkite/scripts/e2e-pipeline/main.py @@ -12,10 +12,10 @@ class BootstrapContextManager: def __enter__(self): - stack_version = os.environ.get("STACK_VERSION") + stack_version = os.environ.get("ELASTIC_STACK_VERSION") project_type = os.environ.get("E2E_PROJECT_TYPE", "on_prems") if stack_version is None: - raise Exception("STACK_VERSION environment variable is missing, please export and try again.") + raise Exception("ELASTIC_STACK_VERSION environment variable is missing, please export and try again.") print(f"Starting E2E test of Logstash running Elastic Integrations against {stack_version} version.") self.bootstrap = Bootstrap(stack_version, project_type) diff --git a/.buildkite/scripts/e2e/plugin_test.py b/.buildkite/scripts/e2e-pipeline/plugin_test.py similarity index 100% rename from .buildkite/scripts/e2e/plugin_test.py rename to .buildkite/scripts/e2e-pipeline/plugin_test.py diff --git a/.buildkite/scripts/e2e/requirements.txt b/.buildkite/scripts/e2e-pipeline/requirements.txt similarity index 100% rename from .buildkite/scripts/e2e/requirements.txt rename to .buildkite/scripts/e2e-pipeline/requirements.txt diff --git a/.buildkite/scripts/e2e/util.py b/.buildkite/scripts/e2e-pipeline/util.py similarity index 100% rename from .buildkite/scripts/e2e/util.py rename to .buildkite/scripts/e2e-pipeline/util.py diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py new file mode 100644 index 00000000..ec57b743 --- /dev/null +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -0,0 +1,98 @@ +import os +import requests +import sys +import typing +from requests.adapters import HTTPAdapter, Retry + +from ruamel.yaml import YAML + +RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" + + +def generate_unit_and_integration_test_steps(stack_version, snapshot) -> list[typing.Any]: + test_steps = [] + + # step-1, unit tests + label_unit_test: typing.final = f"Unit test for {stack_version}, snapshot: {snapshot}" + test_steps.append({ + "label": label_unit_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "INTEGRATION": "false" + } + }) + + # step-2, integration tests + label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}" + test_steps.append({ + "label": label_integration_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "INTEGRATION": "true", + "SECURE_INTEGRATION": "true", + "LOG_LEVEL": "info" + } + }) + return test_steps + + +def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response: + schema = "https://" if "https://" in url else "http://" + session = requests.Session() + # retry on most common failures such as connection timeout(408), etc... + retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504]) + session.mount(schema, HTTPAdapter(max_retries=retries)) + return session.get(url) + + +if __name__ == "__main__": + structure = { + "agents": { + "provider": "gcp", + "machineType": "n1-standard-4", + "image": "family/core-ubuntu-2204" + }, + "steps": []} + + steps = [] + response = call_url_with_retry(RELEASES_URL) + versions_json = response.json() + + target_branch: typing.final = os.getenv("GITHUB_PR_TARGET_BRANCH") + if target_branch == '8.x': + full_stack_version: typing.final = versions_json["snapshots"]["8.future"] + steps += generate_unit_and_integration_test_steps(full_stack_version, "true") + elif target_branch == 'main': + full_stack_version: typing.final = versions_json["snapshots"][target_branch] + steps += generate_unit_and_integration_test_steps(full_stack_version, "true") + else: + # generate steps for the version if released + releases = versions_json["releases"] + for release_version in releases: + if releases[release_version].startswith(target_branch): + steps += generate_unit_and_integration_test_steps(releases[release_version], "false") + break + + # steps for snapshot version + snapshots = versions_json["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith(target_branch): + steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false") + break + + group_desc = f"{target_branch} branch steps" + key_desc = "pr-and-build-steps" + structure["steps"].append({ + "group": group_desc, + "key": key_desc, + "steps": steps + }) + + print( + '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json') + YAML().dump(structure, sys.stdout) diff --git a/.buildkite/scripts/resolve_es_treeish.sh b/.buildkite/scripts/resolve_es_treeish.sh deleted file mode 100755 index 80b6f56f..00000000 --- a/.buildkite/scripts/resolve_es_treeish.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -set +o nounset -if [[ "$SNAPSHOT" == "true" ]] && [[ "$ELASTIC_STACK_VERSION" == "8.x" ]]; then - export ELASTICSEARCH_TREEISH=8.x -else - VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" - - echo "Fetching versions from $VERSION_URL" - VERSIONS=$(curl --retry 5 --retry-delay 5 -fsSL $VERSION_URL) - - if [[ "$SNAPSHOT" == "true" ]]; then - key=$(echo "$VERSIONS" | jq -r '.snapshots."'"$ELASTIC_STACK_VERSION"'"') - else - key=$(echo "$VERSIONS" | jq -r '.releases."'"$ELASTIC_STACK_VERSION"'"') - fi - - export ELASTICSEARCH_TREEISH=${key%.*} -fi diff --git a/.buildkite/scripts/run_e2e_tests.sh b/.buildkite/scripts/run_e2e_tests.sh index a9db3e7f..88ac5e1c 100755 --- a/.buildkite/scripts/run_e2e_tests.sh +++ b/.buildkite/scripts/run_e2e_tests.sh @@ -6,26 +6,6 @@ export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PA eval "$(rbenv init -)" eval "$(pyenv init -)" -VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" - -### -# Resolve stack version and export -resolve_current_stack_version() { - set +o nounset - echo "Fetching versions from $VERSION_URL" - VERSIONS=$(curl --retry 5 --retry-delay 5 -fsSL $VERSION_URL) - - if [[ "$SNAPSHOT" == "true" ]]; then - key=$(echo "$VERSIONS" | jq -r '.snapshots."'"$ELASTIC_STACK_VERSION"'"') - echo "resolved key: $key" - else - key=$(echo "$VERSIONS" | jq -r '.releases."'"$ELASTIC_STACK_VERSION"'"') - fi - - echo "Resolved version: $key" - export STACK_VERSION="$key" -} - ### # Checkout the target branch if defined checkout_target_branch() { @@ -38,6 +18,8 @@ checkout_target_branch() { fi } +### +# Set the Java home based on .java-version set_required_jdk() { set +o nounset java_version="$(cat .java-version)" @@ -73,7 +55,6 @@ build_plugin() { ./gradlew clean vendor localGem } -resolve_current_stack_version checkout_target_branch set_required_jdk build_logstash @@ -81,5 +62,5 @@ build_plugin ### # Install E2E prerequisites and run E2E tests -python3 -mpip install -r .buildkite/scripts/e2e/requirements.txt -python3 .buildkite/scripts/e2e/main.py \ No newline at end of file +python3 -mpip install -r .buildkite/scripts/e2e-pipeline/requirements.txt +python3 .buildkite/scripts/e2e-pipeline/main.py \ No newline at end of file diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index 1108f653..9b8b7257 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1,9 +1,3 @@ -if [ -z "$ELASTICSEARCH_TREEISH" ]; then - source .buildkite/scripts/resolve_es_treeish.sh - echo "Resolved ELASTICSEARCH_TREEISH: ${ELASTICSEARCH_TREEISH}" -else - echo "Using ELASTICSEARCH_TREEISH ${ELASTICSEARCH_TREEISH} defined in the ENV." -fi if [ -z "$TARGET_BRANCH" ]; then echo "Target branch is not specified, using default branch: main or BK defined" diff --git a/gradle.properties b/gradle.properties index 5c258231..89483d25 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ LOGSTASH_PATH=../../logstash -ELASTICSEARCH_TREEISH=8.16 +ELASTICSEARCH_TREEISH=main From 8dd34e336924e86b7411b5684385849656f8b448 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 10 Dec 2024 16:36:11 -0800 Subject: [PATCH 2/4] Generate pull-request CI steps based on the test matrix. --- .../pull-request-pipeline/generate-steps.py | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index ec57b743..e6c4d531 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -7,6 +7,7 @@ from ruamel.yaml import YAML RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_MATRIX_URL = "https://gist.githubusercontent.com/mashhurs/c5b66e05eac1c0968f841e489a8b43a6/raw/eb6b19174977aa4fbe6f7309e35481fb31690e88/tes.yaml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -50,6 +51,11 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque return session.get(url) +def make_matrix_version_key(branch: str) -> str: + branch_parts: typing.final = branch.split(".") + return branch_parts[0] + ".x" + + if __name__ == "__main__": structure = { "agents": { @@ -63,27 +69,28 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque response = call_url_with_retry(RELEASES_URL) versions_json = response.json() + matrix_map = call_url_with_retry(TEST_MATRIX_URL) + matrix_map_yaml = YAML().load(matrix_map.text) + target_branch: typing.final = os.getenv("GITHUB_PR_TARGET_BRANCH") - if target_branch == '8.x': - full_stack_version: typing.final = versions_json["snapshots"]["8.future"] - steps += generate_unit_and_integration_test_steps(full_stack_version, "true") - elif target_branch == 'main': - full_stack_version: typing.final = versions_json["snapshots"][target_branch] - steps += generate_unit_and_integration_test_steps(full_stack_version, "true") - else: - # generate steps for the version if released - releases = versions_json["releases"] - for release_version in releases: - if releases[release_version].startswith(target_branch): - steps += generate_unit_and_integration_test_steps(releases[release_version], "false") - break - - # steps for snapshot version - snapshots = versions_json["snapshots"] - for snapshot_version in snapshots: - if snapshots[snapshot_version].startswith(target_branch): - steps += generate_unit_and_integration_test_steps(snapshots[snapshot_version], "false") - break + matrix_version_key = target_branch if target_branch == "main" else make_matrix_version_key(target_branch) + matrix_releases = matrix_map_yaml.get(matrix_version_key, {}).get("releases", []) + matrix_snapshots = matrix_map_yaml.get(matrix_version_key, {}).get("snapshots", []) + + # let's print what matrix we have got, helps debugging + print(f"matrix_releases: {matrix_releases}") + print(f"matrix_snapshots: {matrix_snapshots}") + for matrix_release in matrix_releases: + full_stack_version: typing.final = versions_json["releases"].get(matrix_release) + # noop, if they are declared in the matrix but not in the release + if full_stack_version is not None: + steps += generate_unit_and_integration_test_steps(full_stack_version, "false") + + for matrix_snapshot in matrix_snapshots: + full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot) + # noop, if they are declared in the matrix but not in the snapshot + if full_stack_version is not None: + steps += generate_unit_and_integration_test_steps(full_stack_version, "true") group_desc = f"{target_branch} branch steps" key_desc = "pr-and-build-steps" From 02644b95fd61ce6c0245742c134a23f18f237782 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Fri, 13 Dec 2024 13:22:58 -0800 Subject: [PATCH 3/4] Apply pull-request test matrix, build and E2E CIs consired manual/auto PR-builds and scheduled runs. --- .buildkite/e2e-pipeline.yml | 4 +- .buildkite/pull-request-pipeline.yml | 2 + .../scripts/build-pipeline/generate-steps.py | 53 ++++++++++---- .../scripts/e2e-pipeline/generate-steps.py | 70 ++++++++++++++----- .../pull-request-pipeline/generate-steps.py | 11 ++- 5 files changed, 104 insertions(+), 36 deletions(-) diff --git a/.buildkite/e2e-pipeline.yml b/.buildkite/e2e-pipeline.yml index 6c2951fc..793a9bdd 100644 --- a/.buildkite/e2e-pipeline.yml +++ b/.buildkite/e2e-pipeline.yml @@ -5,11 +5,11 @@ echo "--- Downloading prerequisites" python3 -m pip install ruamel.yaml - python3 -mpip install -r .buildkite/scripts/e2e-pipeline/requirements.txt + python3 -m pip install requests curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 chmod a+x /usr/bin/yq - echo "--- Generating dynamic steps" + echo "--- Generating steps dynamically" set +e python3 .buildkite/scripts/e2e-pipeline/generate-steps.py > pipeline_steps.yml if [[ $$? -ne 0 ]]; then diff --git a/.buildkite/pull-request-pipeline.yml b/.buildkite/pull-request-pipeline.yml index 658d00ac..80415172 100644 --- a/.buildkite/pull-request-pipeline.yml +++ b/.buildkite/pull-request-pipeline.yml @@ -8,6 +8,8 @@ curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 chmod a+x /usr/bin/yq + export TARGET_BRANCH="$GITHUB_PR_TARGET_BRANCH" + echo "--- Generating steps dynamically" set +e python3 .buildkite/scripts/pull-request-pipeline/generate-steps.py > pipeline_steps.yml diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index 7e7bbc14..b928de15 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -1,3 +1,4 @@ +import os import requests import sys import typing @@ -20,19 +21,42 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque def generate_test_step(stack_version, branch, snapshot) -> dict: label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}" - return { + step: dict = { "label": label_integration_test, "command": TEST_COMMAND, "env": { "SNAPSHOT": snapshot, "ELASTIC_STACK_VERSION": stack_version, - "ELASTICSEARCH_TREEISH": branch, - "TARGET_BRANCH": branch, "INTEGRATION": "true", "SECURE_INTEGRATION": "true", "LOG_LEVEL": "info" } } + # we are not going to set branch if job kicked of through webhook (PR merge or manual PR run) + if branch is not None: + step["env"]["TARGET_BRANCH"] = branch + return step + + +def generate_steps_for_scheduler(versions) -> list: + steps: list = [] + snapshots = versions["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith("7."): + continue + full_stack_version = snapshots[snapshot_version] + version_parts = snapshots[snapshot_version].split(".") + major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" + branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions + steps.append(generate_test_step(full_stack_version, branch, "true")) + return steps + + +def generate_steps_for_main_branch(versions) -> list: + steps: list = [] + full_stack_version: typing.final = versions["snapshots"]["main"] + steps += generate_test_step(full_stack_version, None, "true") + return steps if __name__ == "__main__": @@ -44,18 +68,19 @@ def generate_test_step(stack_version, branch, snapshot) -> dict: }, "steps": []} - steps = [] response = call_url_with_retry(RELEASES_URL) - versions_json = response.json() - snapshots = versions_json["snapshots"] - for snapshot_version in snapshots: - if snapshots[snapshot_version].startswith("7.") or snapshots[snapshot_version].startswith("8.15"): - continue - full_stack_version = snapshots[snapshot_version] - version_parts = snapshots[snapshot_version].split(".") - major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" - branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions - steps.append(generate_test_step(full_stack_version, branch, "true")) + versions_json: typing.final = response.json() + + # Use BUILDKITE_SOURCE to figure out PR merge or schedule. + # If PR merge, no need to run builds on all branches, target branch will be good + # - webhook when PR gets merged + # - schedule when daily schedule starts + # - ui when manually kicking job from BK UI + # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH + bk_source = os.getenv("BUILDKITE_SOURCE") + bk_branch = os.getenv("BUILDKITE_BRANCH") + steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_json) group_desc = f"Build steps" key_desc = "build-steps" diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py index 92b3986f..e9fc6166 100644 --- a/.buildkite/scripts/e2e-pipeline/generate-steps.py +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -1,25 +1,58 @@ +import os +import requests import sys import typing -import util - +from requests.adapters import HTTPAdapter, Retry from ruamel.yaml import YAML RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh" -def generate_test_step(stack_version, es_treeish, snapshot) -> dict: +def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response: + schema = "https://" if "https://" in url else "http://" + session = requests.Session() + # retry on most common failures such as connection timeout(408), etc... + retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504]) + session.mount(schema, HTTPAdapter(max_retries=retries)) + return session.get(url) + + +def generate_test_step(stack_version, branch, snapshot) -> dict: label_integration_test: typing.final = f"E2E tests for {stack_version}, snapshot: {snapshot}" - return { + step: dict = { "label": label_integration_test, "command": TEST_COMMAND, "env": { "SNAPSHOT": snapshot, "ELASTIC_STACK_VERSION": stack_version, - "ELASTICSEARCH_TREEISH": es_treeish, - "TARGET_BRANCH": branch } } + # we are not going to set branch if job kicked of through webhook (PR merge or manual PR run) + if branch is not None: + step["env"]["TARGET_BRANCH"] = branch + return step + + +def generate_steps_for_scheduler(versions) -> list: + steps: list = [] + snapshots = versions["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith("7."): + continue + full_stack_version = snapshots[snapshot_version] + version_parts = snapshots[snapshot_version].split(".") + major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" + branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions + steps.append(generate_test_step(full_stack_version, branch, "true")) + return steps + + +def generate_steps_for_main_branch(versions) -> list: + steps: list = [] + full_stack_version: typing.final = versions["snapshots"]["main"] + steps.append(generate_test_step(full_stack_version, None, "true")) + return steps if __name__ == "__main__": @@ -33,18 +66,19 @@ def generate_test_step(stack_version, es_treeish, snapshot) -> dict: }, "steps": []} - steps = [] - response = util.call_url_with_retry(RELEASES_URL) - release_json = response.json() - snapshots = release_json["snapshots"] - for snapshot_version in snapshots: - if snapshots[snapshot_version].startswith("7.") or snapshots[snapshot_version].startswith("8.15"): - continue - full_stack_version = snapshots[snapshot_version] - version_parts = snapshots[snapshot_version].split(".") - major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" - branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions - steps.append(generate_test_step(full_stack_version, branch, "true")) + response = call_url_with_retry(RELEASES_URL) + versions_json: typing.final = response.json() + + # Use BUILDKITE_SOURCE to figure out PR merge or schedule. + # If PR merge, no need to run builds on all branches, target branch will be good + # - webhook when PR gets merged + # - schedule when daily schedule starts + # - ui when manually kicking job from BK UI + # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH + bk_source = os.getenv("BUILDKITE_SOURCE") + bk_branch = os.getenv("BUILDKITE_BRANCH") + steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_json) group_desc = f"E2E steps" key_desc = "e2e-steps" diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py index e6c4d531..5a4c7cab 100644 --- a/.buildkite/scripts/pull-request-pipeline/generate-steps.py +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -7,7 +7,8 @@ from ruamel.yaml import YAML RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" -TEST_MATRIX_URL = "https://gist.githubusercontent.com/mashhurs/c5b66e05eac1c0968f841e489a8b43a6/raw/eb6b19174977aa4fbe6f7309e35481fb31690e88/tes.yaml" +TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \ + "-request-test-matrix.yml" TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" @@ -72,7 +73,13 @@ def make_matrix_version_key(branch: str) -> str: matrix_map = call_url_with_retry(TEST_MATRIX_URL) matrix_map_yaml = YAML().load(matrix_map.text) - target_branch: typing.final = os.getenv("GITHUB_PR_TARGET_BRANCH") + # there are situations to manually run CIs with PR change, + # set MANUAL_TARGET_BRANCH with upstream target branch and run + manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH") + target_branch: typing.final = manually_set_target_branch if manually_set_target_branch \ + else os.getenv("TARGET_BRANCH") + print(f"Running with target_branch: {target_branch}") + matrix_version_key = target_branch if target_branch == "main" else make_matrix_version_key(target_branch) matrix_releases = matrix_map_yaml.get(matrix_version_key, {}).get("releases", []) matrix_snapshots = matrix_map_yaml.get(matrix_version_key, {}).get("snapshots", []) From 2f569b06be22124cd9958f40590fc55728b8cd8d Mon Sep 17 00:00:00 2001 From: Mashhur Date: Fri, 13 Dec 2024 13:34:37 -0800 Subject: [PATCH 4/4] Build pipeline steps generation fix. --- .buildkite/scripts/build-pipeline/generate-steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py index b928de15..bf74f560 100644 --- a/.buildkite/scripts/build-pipeline/generate-steps.py +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -55,7 +55,7 @@ def generate_steps_for_scheduler(versions) -> list: def generate_steps_for_main_branch(versions) -> list: steps: list = [] full_stack_version: typing.final = versions["snapshots"]["main"] - steps += generate_test_step(full_stack_version, None, "true") + steps.append(generate_test_step(full_stack_version, None, "true")) return steps