diff --git a/.buildkite/build-pipeline.yml b/.buildkite/build-pipeline.yml index dbf9ee0c..538dfb0b 100644 --- a/.buildkite/build-pipeline.yml +++ b/.buildkite/build-pipeline.yml @@ -1,43 +1,28 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +- label: "Build pipeline" + command: | + #!/usr/bin/env bash + set -eo pipefail -agents: - provider: "gcp" - machineType: "n1-standard-4" - image: family/core-ubuntu-2204 - -steps: - - label: ":hammer: Build plugin with LS & ES 8.x :elasticsearch:" - # Builds with LS and ES last 8.x released version - # Runs integration tests on 8.x released versions - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - SNAPSHOT: false - INTEGRATION: true - SECURE_INTEGRATION: true - TARGET_BRANCH: "8.x" - - - label: ":hammer: Build plugin with LS 8.x-SNAPSHOT & ES `main` branch :elasticsearch:" - # Builds with LS last 8.x released version and ES main - # Runs integration tests on 8.x released versions - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - ELASTICSEARCH_TREEISH: "main" - SNAPSHOT: true - INTEGRATION: true - SECURE_INTEGRATION: true - - - label: ":hammer: Build plugin with LS & ES `main` branch :elasticsearch:" - # Builds with LS last 8.x released version and ES main - # Runs integration tests on 8.x released versions - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: "main" - ELASTICSEARCH_TREEISH: "main" - SNAPSHOT: true - INTEGRATION: true - SECURE_INTEGRATION: true \ No newline at end of file + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + python3 -m pip install requests + curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod a+x /usr/bin/yq + + echo "--- Generating dynamic steps" + set +e + python3 .buildkite/scripts/build-pipeline/generate-steps.py > pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem with generating pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml | yq . + fi + + set -eo pipefail + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/e2e-pipeline.yml b/.buildkite/e2e-pipeline.yml index 85bd871e..793a9bdd 100644 --- a/.buildkite/e2e-pipeline.yml +++ b/.buildkite/e2e-pipeline.yml @@ -1,39 +1,28 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +- label: "E2E pipeline" + command: | + #!/usr/bin/env bash + set -eo pipefail -agents: - provider: gcp - imageProject: elastic-images-prod - image: family/platform-ingest-logstash-multi-jdk-ubuntu-2204 - machineType: "n2-standard-4" - diskSizeGb: 120 - -steps: - # ------------- Run E2E tests --------------------- - - label: ":test_tube: Run E2E tests with LS 8.x :rocket:" - # uses the LS main & plugin 8.x branch when building the plugin - # Runs integration tests against 8.x release version - command: - - .buildkite/scripts/run_e2e_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - TARGET_BRANCH: "8.x" - - - label: ":test_tube: Run E2E tests with LS 8.x-SNAPSHOT :rocket:" - # uses the LS & plugin main branch when building the plugin - # Runs integration tests against 8.x-SNAPSHOT version - command: - - .buildkite/scripts/run_e2e_tests.sh - env: - ELASTIC_STACK_VERSION: "8.current" - SNAPSHOT: true - TARGET_BRANCH: "8.x" - - - label: ":test_tube: Run E2E tests with LS `main` :rocket:" - # uses the LS & plugin main branch when building the plugin - # Runs integration tests against snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json - command: - - .buildkite/scripts/run_e2e_tests.sh - env: - ELASTIC_STACK_VERSION: main - SNAPSHOT: true - ELASTICSEARCH_TREEISH: main \ No newline at end of file + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + python3 -m pip install requests + curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod a+x /usr/bin/yq + + echo "--- Generating steps dynamically" + set +e + python3 .buildkite/scripts/e2e-pipeline/generate-steps.py > pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem with generating pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml | yq . + fi + + set -eo pipefail + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/pull-request-pipeline.yml b/.buildkite/pull-request-pipeline.yml index ca7396a1..80415172 100644 --- a/.buildkite/pull-request-pipeline.yml +++ b/.buildkite/pull-request-pipeline.yml @@ -1,32 +1,29 @@ -# define a GCP VM agent to support container management (by default agent doesn't support) -agents: - provider: "gcp" - machineType: "n1-standard-4" - image: family/core-ubuntu-2204 +- label: "Pull request pipeline" + command: | + #!/usr/bin/env bash + set -eo pipefail + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + python3 -m pip install requests + curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + chmod a+x /usr/bin/yq -steps: - # ------------- Unit tests --------------------- - - label: ":hammer: Unit tests with LS & ES main :docker:" - # Builds the plugin (with current changes) against LS and ES main - # Runs unit tests on LS & ES main docker - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: main - ELASTICSEARCH_TREEISH: main - INTEGRATION: false - SNAPSHOT: true + export TARGET_BRANCH="$GITHUB_PR_TARGET_BRANCH" - # ------------- Integration tests --------------------- - - label: ":hammer: Integration tests with LS & ES main :docker:" - # Builds the plugin (with current changes) against LS and ES main - # Runs integration tests on snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json - command: - - .buildkite/scripts/run_tests.sh - env: - ELASTIC_STACK_VERSION: main - ELASTICSEARCH_TREEISH: main - SNAPSHOT: true - INTEGRATION: true - SECURE_INTEGRATION: true - LOG_LEVEL: "info" \ No newline at end of file + echo "--- Generating steps dynamically" + set +e + python3 .buildkite/scripts/pull-request-pipeline/generate-steps.py > pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem with generating pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml | yq . + fi + + set -eo pipefail + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/build-pipeline/generate-steps.py b/.buildkite/scripts/build-pipeline/generate-steps.py new file mode 100644 index 00000000..bf74f560 --- /dev/null +++ b/.buildkite/scripts/build-pipeline/generate-steps.py @@ -0,0 +1,95 @@ +import os +import requests +import sys +import typing +from requests.adapters import HTTPAdapter, Retry + +from ruamel.yaml import YAML + +RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" + + +def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response: + schema = "https://" if "https://" in url else "http://" + session = requests.Session() + # retry on most common failures such as connection timeout(408), etc... + retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504]) + session.mount(schema, HTTPAdapter(max_retries=retries)) + return session.get(url) + + +def generate_test_step(stack_version, branch, snapshot) -> dict: + label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}" + step: dict = { + "label": label_integration_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "INTEGRATION": "true", + "SECURE_INTEGRATION": "true", + "LOG_LEVEL": "info" + } + } + # we are not going to set branch if job kicked of through webhook (PR merge or manual PR run) + if branch is not None: + step["env"]["TARGET_BRANCH"] = branch + return step + + +def generate_steps_for_scheduler(versions) -> list: + steps: list = [] + snapshots = versions["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith("7."): + continue + full_stack_version = snapshots[snapshot_version] + version_parts = snapshots[snapshot_version].split(".") + major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" + branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions + steps.append(generate_test_step(full_stack_version, branch, "true")) + return steps + + +def generate_steps_for_main_branch(versions) -> list: + steps: list = [] + full_stack_version: typing.final = versions["snapshots"]["main"] + steps.append(generate_test_step(full_stack_version, None, "true")) + return steps + + +if __name__ == "__main__": + structure = { + "agents": { + "provider": "gcp", + "machineType": "n1-standard-4", + "image": "family/core-ubuntu-2204" + }, + "steps": []} + + response = call_url_with_retry(RELEASES_URL) + versions_json: typing.final = response.json() + + # Use BUILDKITE_SOURCE to figure out PR merge or schedule. + # If PR merge, no need to run builds on all branches, target branch will be good + # - webhook when PR gets merged + # - schedule when daily schedule starts + # - ui when manually kicking job from BK UI + # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH + bk_source = os.getenv("BUILDKITE_SOURCE") + bk_branch = os.getenv("BUILDKITE_BRANCH") + steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_json) + + group_desc = f"Build steps" + key_desc = "build-steps" + structure["steps"].append({ + "group": group_desc, + "key": key_desc, + "steps": steps, + }) + + print( + '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json') + YAML().dump(structure, sys.stdout) diff --git a/.buildkite/scripts/e2e/README.md b/.buildkite/scripts/e2e-pipeline/README.md similarity index 89% rename from .buildkite/scripts/e2e/README.md rename to .buildkite/scripts/e2e-pipeline/README.md index 70ca9d9c..f62be8fd 100644 --- a/.buildkite/scripts/e2e/README.md +++ b/.buildkite/scripts/e2e-pipeline/README.md @@ -32,19 +32,19 @@ In order to run tests with serverless, you also need to export `EC_API_KEY` whic In the pipelines, this will be automatically retrieved from Vault services. #### Stack version -E2E also requires `STACK_VERSION` (ex: "8.12.0") environment variable in order to test against. +E2E also requires `ELASTIC_STACK_VERSION` (ex: "8.12.0") environment variable in order to test against. Make sure to export it before running. In the Buildkite pipeline, this var will be resolved and exported. #### Installing dependencies Make sure you have python installed on you local ```bash -pip install -r .buildkite/scripts/e2e/requirements.txt +pip install -r .buildkite/scripts/e2e-pipeline/requirements.txt ``` ### Run Run the following command from the repo dir: ```bash -python3 .buildkite/scripts/e2e/main.py +python3 .buildkite/scripts/e2e-pipeline/main.py ``` ## Troubleshooting diff --git a/.buildkite/scripts/e2e/__init__.py b/.buildkite/scripts/e2e-pipeline/__init__.py similarity index 100% rename from .buildkite/scripts/e2e/__init__.py rename to .buildkite/scripts/e2e-pipeline/__init__.py diff --git a/.buildkite/scripts/e2e/bootstrap.py b/.buildkite/scripts/e2e-pipeline/bootstrap.py similarity index 99% rename from .buildkite/scripts/e2e/bootstrap.py rename to .buildkite/scripts/e2e-pipeline/bootstrap.py index 17bb8f9b..b70110b5 100644 --- a/.buildkite/scripts/e2e/bootstrap.py +++ b/.buildkite/scripts/e2e-pipeline/bootstrap.py @@ -122,7 +122,7 @@ def __reload_container(self) -> None: time.sleep(20) # give a time Logstash pipeline to fully start def __update_pipeline_config(self) -> None: - local_config_file_path = ".buildkite/scripts/e2e/config/" + local_config_file_path = ".buildkite/scripts/e2e-pipeline/config/" config_file = "serverless_pipeline.conf" if self.project_type == "serverless" else "pipeline.conf" local_config_file = local_config_file_path + config_file container_config_file_path = "/usr/share/logstash/pipeline/logstash.conf" diff --git a/.buildkite/scripts/e2e/config/pipeline.conf b/.buildkite/scripts/e2e-pipeline/config/pipeline.conf similarity index 100% rename from .buildkite/scripts/e2e/config/pipeline.conf rename to .buildkite/scripts/e2e-pipeline/config/pipeline.conf diff --git a/.buildkite/scripts/e2e/config/serverless_pipeline.conf b/.buildkite/scripts/e2e-pipeline/config/serverless_pipeline.conf similarity index 100% rename from .buildkite/scripts/e2e/config/serverless_pipeline.conf rename to .buildkite/scripts/e2e-pipeline/config/serverless_pipeline.conf diff --git a/.buildkite/scripts/e2e-pipeline/generate-steps.py b/.buildkite/scripts/e2e-pipeline/generate-steps.py new file mode 100644 index 00000000..e9fc6166 --- /dev/null +++ b/.buildkite/scripts/e2e-pipeline/generate-steps.py @@ -0,0 +1,93 @@ +import os +import requests +import sys +import typing +from requests.adapters import HTTPAdapter, Retry +from ruamel.yaml import YAML + +RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh" + + +def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response: + schema = "https://" if "https://" in url else "http://" + session = requests.Session() + # retry on most common failures such as connection timeout(408), etc... + retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504]) + session.mount(schema, HTTPAdapter(max_retries=retries)) + return session.get(url) + + +def generate_test_step(stack_version, branch, snapshot) -> dict: + label_integration_test: typing.final = f"E2E tests for {stack_version}, snapshot: {snapshot}" + step: dict = { + "label": label_integration_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + } + } + # we are not going to set branch if job kicked of through webhook (PR merge or manual PR run) + if branch is not None: + step["env"]["TARGET_BRANCH"] = branch + return step + + +def generate_steps_for_scheduler(versions) -> list: + steps: list = [] + snapshots = versions["snapshots"] + for snapshot_version in snapshots: + if snapshots[snapshot_version].startswith("7."): + continue + full_stack_version = snapshots[snapshot_version] + version_parts = snapshots[snapshot_version].split(".") + major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}" + branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions + steps.append(generate_test_step(full_stack_version, branch, "true")) + return steps + + +def generate_steps_for_main_branch(versions) -> list: + steps: list = [] + full_stack_version: typing.final = versions["snapshots"]["main"] + steps.append(generate_test_step(full_stack_version, None, "true")) + return steps + + +if __name__ == "__main__": + structure = { + "agents": { + "provider": "gcp", + "machineType": "n2-standard-4", + "imageProject": "elastic-images-prod", + "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204", + "diskSizeGb": 120 + }, + "steps": []} + + response = call_url_with_retry(RELEASES_URL) + versions_json: typing.final = response.json() + + # Use BUILDKITE_SOURCE to figure out PR merge or schedule. + # If PR merge, no need to run builds on all branches, target branch will be good + # - webhook when PR gets merged + # - schedule when daily schedule starts + # - ui when manually kicking job from BK UI + # - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH + bk_source = os.getenv("BUILDKITE_SOURCE") + bk_branch = os.getenv("BUILDKITE_BRANCH") + steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \ + else generate_steps_for_main_branch(versions_json) + + group_desc = f"E2E steps" + key_desc = "e2e-steps" + structure["steps"].append({ + "group": group_desc, + "key": key_desc, + "steps": steps, + }) + + print( + '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json') + YAML().dump(structure, sys.stdout) diff --git a/.buildkite/scripts/e2e/logstash_stats.py b/.buildkite/scripts/e2e-pipeline/logstash_stats.py similarity index 100% rename from .buildkite/scripts/e2e/logstash_stats.py rename to .buildkite/scripts/e2e-pipeline/logstash_stats.py diff --git a/.buildkite/scripts/e2e/main.py b/.buildkite/scripts/e2e-pipeline/main.py similarity index 90% rename from .buildkite/scripts/e2e/main.py rename to .buildkite/scripts/e2e-pipeline/main.py index 09e2c41e..de6d5ca6 100644 --- a/.buildkite/scripts/e2e/main.py +++ b/.buildkite/scripts/e2e-pipeline/main.py @@ -12,10 +12,10 @@ class BootstrapContextManager: def __enter__(self): - stack_version = os.environ.get("STACK_VERSION") + stack_version = os.environ.get("ELASTIC_STACK_VERSION") project_type = os.environ.get("E2E_PROJECT_TYPE", "on_prems") if stack_version is None: - raise Exception("STACK_VERSION environment variable is missing, please export and try again.") + raise Exception("ELASTIC_STACK_VERSION environment variable is missing, please export and try again.") print(f"Starting E2E test of Logstash running Elastic Integrations against {stack_version} version.") self.bootstrap = Bootstrap(stack_version, project_type) diff --git a/.buildkite/scripts/e2e/plugin_test.py b/.buildkite/scripts/e2e-pipeline/plugin_test.py similarity index 100% rename from .buildkite/scripts/e2e/plugin_test.py rename to .buildkite/scripts/e2e-pipeline/plugin_test.py diff --git a/.buildkite/scripts/e2e/requirements.txt b/.buildkite/scripts/e2e-pipeline/requirements.txt similarity index 100% rename from .buildkite/scripts/e2e/requirements.txt rename to .buildkite/scripts/e2e-pipeline/requirements.txt diff --git a/.buildkite/scripts/e2e/util.py b/.buildkite/scripts/e2e-pipeline/util.py similarity index 100% rename from .buildkite/scripts/e2e/util.py rename to .buildkite/scripts/e2e-pipeline/util.py diff --git a/.buildkite/scripts/pull-request-pipeline/generate-steps.py b/.buildkite/scripts/pull-request-pipeline/generate-steps.py new file mode 100644 index 00000000..5a4c7cab --- /dev/null +++ b/.buildkite/scripts/pull-request-pipeline/generate-steps.py @@ -0,0 +1,112 @@ +import os +import requests +import sys +import typing +from requests.adapters import HTTPAdapter, Retry + +from ruamel.yaml import YAML + +RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" +TEST_MATRIX_URL = "https://raw.githubusercontent.com/elastic/logstash-filter-elastic_integration/main/.buildkite/pull" \ + "-request-test-matrix.yml" +TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh" + + +def generate_unit_and_integration_test_steps(stack_version, snapshot) -> list[typing.Any]: + test_steps = [] + + # step-1, unit tests + label_unit_test: typing.final = f"Unit test for {stack_version}, snapshot: {snapshot}" + test_steps.append({ + "label": label_unit_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "INTEGRATION": "false" + } + }) + + # step-2, integration tests + label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}" + test_steps.append({ + "label": label_integration_test, + "command": TEST_COMMAND, + "env": { + "SNAPSHOT": snapshot, + "ELASTIC_STACK_VERSION": stack_version, + "INTEGRATION": "true", + "SECURE_INTEGRATION": "true", + "LOG_LEVEL": "info" + } + }) + return test_steps + + +def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response: + schema = "https://" if "https://" in url else "http://" + session = requests.Session() + # retry on most common failures such as connection timeout(408), etc... + retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504]) + session.mount(schema, HTTPAdapter(max_retries=retries)) + return session.get(url) + + +def make_matrix_version_key(branch: str) -> str: + branch_parts: typing.final = branch.split(".") + return branch_parts[0] + ".x" + + +if __name__ == "__main__": + structure = { + "agents": { + "provider": "gcp", + "machineType": "n1-standard-4", + "image": "family/core-ubuntu-2204" + }, + "steps": []} + + steps = [] + response = call_url_with_retry(RELEASES_URL) + versions_json = response.json() + + matrix_map = call_url_with_retry(TEST_MATRIX_URL) + matrix_map_yaml = YAML().load(matrix_map.text) + + # there are situations to manually run CIs with PR change, + # set MANUAL_TARGET_BRANCH with upstream target branch and run + manually_set_target_branch: typing.final = os.getenv("MANUAL_TARGET_BRANCH") + target_branch: typing.final = manually_set_target_branch if manually_set_target_branch \ + else os.getenv("TARGET_BRANCH") + print(f"Running with target_branch: {target_branch}") + + matrix_version_key = target_branch if target_branch == "main" else make_matrix_version_key(target_branch) + matrix_releases = matrix_map_yaml.get(matrix_version_key, {}).get("releases", []) + matrix_snapshots = matrix_map_yaml.get(matrix_version_key, {}).get("snapshots", []) + + # let's print what matrix we have got, helps debugging + print(f"matrix_releases: {matrix_releases}") + print(f"matrix_snapshots: {matrix_snapshots}") + for matrix_release in matrix_releases: + full_stack_version: typing.final = versions_json["releases"].get(matrix_release) + # noop, if they are declared in the matrix but not in the release + if full_stack_version is not None: + steps += generate_unit_and_integration_test_steps(full_stack_version, "false") + + for matrix_snapshot in matrix_snapshots: + full_stack_version: typing.final = versions_json["snapshots"].get(matrix_snapshot) + # noop, if they are declared in the matrix but not in the snapshot + if full_stack_version is not None: + steps += generate_unit_and_integration_test_steps(full_stack_version, "true") + + group_desc = f"{target_branch} branch steps" + key_desc = "pr-and-build-steps" + structure["steps"].append({ + "group": group_desc, + "key": key_desc, + "steps": steps + }) + + print( + '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json') + YAML().dump(structure, sys.stdout) diff --git a/.buildkite/scripts/resolve_es_treeish.sh b/.buildkite/scripts/resolve_es_treeish.sh deleted file mode 100755 index 80b6f56f..00000000 --- a/.buildkite/scripts/resolve_es_treeish.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -set +o nounset -if [[ "$SNAPSHOT" == "true" ]] && [[ "$ELASTIC_STACK_VERSION" == "8.x" ]]; then - export ELASTICSEARCH_TREEISH=8.x -else - VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" - - echo "Fetching versions from $VERSION_URL" - VERSIONS=$(curl --retry 5 --retry-delay 5 -fsSL $VERSION_URL) - - if [[ "$SNAPSHOT" == "true" ]]; then - key=$(echo "$VERSIONS" | jq -r '.snapshots."'"$ELASTIC_STACK_VERSION"'"') - else - key=$(echo "$VERSIONS" | jq -r '.releases."'"$ELASTIC_STACK_VERSION"'"') - fi - - export ELASTICSEARCH_TREEISH=${key%.*} -fi diff --git a/.buildkite/scripts/run_e2e_tests.sh b/.buildkite/scripts/run_e2e_tests.sh index a9db3e7f..88ac5e1c 100755 --- a/.buildkite/scripts/run_e2e_tests.sh +++ b/.buildkite/scripts/run_e2e_tests.sh @@ -6,26 +6,6 @@ export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PA eval "$(rbenv init -)" eval "$(pyenv init -)" -VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json" - -### -# Resolve stack version and export -resolve_current_stack_version() { - set +o nounset - echo "Fetching versions from $VERSION_URL" - VERSIONS=$(curl --retry 5 --retry-delay 5 -fsSL $VERSION_URL) - - if [[ "$SNAPSHOT" == "true" ]]; then - key=$(echo "$VERSIONS" | jq -r '.snapshots."'"$ELASTIC_STACK_VERSION"'"') - echo "resolved key: $key" - else - key=$(echo "$VERSIONS" | jq -r '.releases."'"$ELASTIC_STACK_VERSION"'"') - fi - - echo "Resolved version: $key" - export STACK_VERSION="$key" -} - ### # Checkout the target branch if defined checkout_target_branch() { @@ -38,6 +18,8 @@ checkout_target_branch() { fi } +### +# Set the Java home based on .java-version set_required_jdk() { set +o nounset java_version="$(cat .java-version)" @@ -73,7 +55,6 @@ build_plugin() { ./gradlew clean vendor localGem } -resolve_current_stack_version checkout_target_branch set_required_jdk build_logstash @@ -81,5 +62,5 @@ build_plugin ### # Install E2E prerequisites and run E2E tests -python3 -mpip install -r .buildkite/scripts/e2e/requirements.txt -python3 .buildkite/scripts/e2e/main.py \ No newline at end of file +python3 -mpip install -r .buildkite/scripts/e2e-pipeline/requirements.txt +python3 .buildkite/scripts/e2e-pipeline/main.py \ No newline at end of file diff --git a/.buildkite/scripts/run_tests.sh b/.buildkite/scripts/run_tests.sh index 1108f653..9b8b7257 100755 --- a/.buildkite/scripts/run_tests.sh +++ b/.buildkite/scripts/run_tests.sh @@ -1,9 +1,3 @@ -if [ -z "$ELASTICSEARCH_TREEISH" ]; then - source .buildkite/scripts/resolve_es_treeish.sh - echo "Resolved ELASTICSEARCH_TREEISH: ${ELASTICSEARCH_TREEISH}" -else - echo "Using ELASTICSEARCH_TREEISH ${ELASTICSEARCH_TREEISH} defined in the ENV." -fi if [ -z "$TARGET_BRANCH" ]; then echo "Target branch is not specified, using default branch: main or BK defined" diff --git a/gradle.properties b/gradle.properties index 5c258231..89483d25 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ LOGSTASH_PATH=../../logstash -ELASTICSEARCH_TREEISH=8.16 +ELASTICSEARCH_TREEISH=main