[main branch] Rebuilds pull-request, build and E2E pipelines.

.buildkite/build-pipeline.yml

@@ -1,43 +1,28 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+- label: "Build pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
 
-agents:
-  provider: "gcp"
-  machineType: "n1-standard-4"
-  image: family/core-ubuntu-2204
-
-steps:
-  - label: ":hammer: Build plugin with LS & ES 8.x :elasticsearch:"
-    # Builds with LS and ES last 8.x released version
-    # Runs integration tests on 8.x released versions
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      SNAPSHOT: false
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-      TARGET_BRANCH: "8.x"
-
-  - label: ":hammer: Build plugin with LS 8.x-SNAPSHOT & ES `main` branch :elasticsearch:"
-    # Builds with LS last 8.x released version and ES main
-    # Runs integration tests on 8.x released versions
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      ELASTICSEARCH_TREEISH: "main"
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-
-  - label: ":hammer: Build plugin with LS & ES `main` branch :elasticsearch:"
-    # Builds with LS last 8.x released version and ES main
-    # Runs integration tests on 8.x released versions
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "main"
-      ELASTICSEARCH_TREEISH: "main"
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+
+    echo "--- Generating dynamic steps"
+    set +e
+    python3 .buildkite/scripts/build-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/e2e-pipeline.yml

@@ -1,39 +1,28 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+- label: "E2E pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
 
-agents:
-  provider: gcp
-  imageProject: elastic-images-prod
-  image: family/platform-ingest-logstash-multi-jdk-ubuntu-2204
-  machineType: "n2-standard-4"
-  diskSizeGb: 120
-
-steps:
-  # ------------- Run E2E tests ---------------------
-  - label: ":test_tube: Run E2E tests with LS 8.x :rocket:"
-    # uses the LS main & plugin 8.x branch when building the plugin
-    # Runs integration tests against 8.x release version
-    command:
-      - .buildkite/scripts/run_e2e_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      TARGET_BRANCH: "8.x"
-
-  - label: ":test_tube: Run E2E tests with LS 8.x-SNAPSHOT :rocket:"
-    # uses the LS & plugin main branch when building the plugin
-    # Runs integration tests against 8.x-SNAPSHOT version
-    command:
-      - .buildkite/scripts/run_e2e_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      SNAPSHOT: true
-      TARGET_BRANCH: "8.x"
-
-  - label: ":test_tube: Run E2E tests with LS `main` :rocket:"
-    # uses the LS & plugin main branch when building the plugin
-    # Runs integration tests against snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json
-    command:
-      - .buildkite/scripts/run_e2e_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: main
-      SNAPSHOT: true
-      ELASTICSEARCH_TREEISH: main
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -mpip install -r .buildkite/scripts/e2e-pipeline/requirements.txt
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+
+    echo "--- Generating dynamic steps"
+    set +e
+    python3 .buildkite/scripts/e2e-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/pull-request-pipeline.yml

@@ -1,32 +1,27 @@
-# define a GCP VM agent to support container management (by default agent doesn't support)
-agents:
-  provider: "gcp"
-  machineType: "n1-standard-4"
-  image: family/core-ubuntu-2204
+- label: "Pull request pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
 
-steps:
-  # ------------- Unit tests ---------------------
-  - label: ":hammer: Unit tests with LS & ES main :docker:"
-    # Builds the plugin (with current changes) against LS and ES main
-    # Runs unit tests on LS & ES main docker
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: main
-      ELASTICSEARCH_TREEISH: main
-      INTEGRATION: false
-      SNAPSHOT: true
-
-  # ------------- Integration tests ---------------------
-  - label: ":hammer: Integration tests with LS & ES main :docker:"
-    # Builds the plugin (with current changes) against LS and ES main
-    # Runs integration tests on snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: main
-      ELASTICSEARCH_TREEISH: main
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-      LOG_LEVEL: "info"
+    echo "--- Generating steps dynamically"
+    set +e
+    python3 .buildkite/scripts/pull-request-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/scripts/build-pipeline/generate-steps.py

@@ -0,0 +1,70 @@
+import requests
+import sys
+import typing
+from requests.adapters import HTTPAdapter, Retry
+
+from ruamel.yaml import YAML
+
+RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
+
+
+def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response:
+    schema = "https://" if "https://" in url else "http://"
+    session = requests.Session()
+    # retry on most common failures such as connection timeout(408), etc...
+    retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504])
+    session.mount(schema, HTTPAdapter(max_retries=retries))
+    return session.get(url)
+
+
+def generate_test_step(stack_version, branch, snapshot) -> dict:
+    label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}"
+    return {
+        "label": label_integration_test,
+        "command": TEST_COMMAND,
+        "env": {
+            "SNAPSHOT": snapshot,
+            "ELASTIC_STACK_VERSION": stack_version,
+            "ELASTICSEARCH_TREEISH": branch,
+            "TARGET_BRANCH": branch,
+            "INTEGRATION": "true",
+            "SECURE_INTEGRATION": "true",
+            "LOG_LEVEL": "info"
+        }
+    }
+
+
+if __name__ == "__main__":
+    structure = {
+        "agents": {
+            "provider": "gcp",
+            "machineType": "n1-standard-4",
+            "image": "family/core-ubuntu-2204"
+        },
+        "steps": []}
+
+    steps = []
+    response = call_url_with_retry(RELEASES_URL)
+    versions_json = response.json()
+    snapshots = versions_json["snapshots"]
+    for snapshot_version in snapshots:
+        if snapshots[snapshot_version].startswith("7.") or snapshots[snapshot_version].startswith("8.15"):
+            continue
+        full_stack_version = snapshots[snapshot_version]
+        version_parts = snapshots[snapshot_version].split(".")
+        major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}"
+        branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions
+        steps.append(generate_test_step(full_stack_version, branch, "true"))
+
+    group_desc = f"Build steps"
+    key_desc = "build-steps"
+    structure["steps"].append({
+        "group": group_desc,
+        "key": key_desc,
+        "steps": steps,
+    })
+
+    print(
+        '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
+    YAML().dump(structure, sys.stdout)

.buildkite/scripts/e2e/README.md → .buildkite/scripts/e2e-pipeline/README.md

@@ -32,19 +32,19 @@ In order to run tests with serverless, you also need to export `EC_API_KEY` whic
 In the pipelines, this will be automatically retrieved from Vault services.
 
 #### Stack version
-E2E also requires `STACK_VERSION` (ex: "8.12.0") environment variable in order to test against.
+E2E also requires `ELASTIC_STACK_VERSION` (ex: "8.12.0") environment variable in order to test against.
 Make sure to export it before running. In the Buildkite pipeline, this var will be resolved and exported. 
 
 #### Installing dependencies
 Make sure you have python installed on you local
 ```bash
-pip install -r .buildkite/scripts/e2e/requirements.txt
+pip install -r .buildkite/scripts/e2e-pipeline/requirements.txt
 ```
 
 ### Run
 Run the following command from the repo dir:
 ```bash
-python3 .buildkite/scripts/e2e/main.py
+python3 .buildkite/scripts/e2e-pipeline/main.py
 ```
 
 ## Troubleshooting

.buildkite/scripts/e2e/bootstrap.py → .buildkite/scripts/e2e-pipeline/bootstrap.py

@@ -122,7 +122,7 @@ def __reload_container(self) -> None:
         time.sleep(20)  # give a time Logstash pipeline to fully start
 
     def __update_pipeline_config(self) -> None:
-        local_config_file_path = ".buildkite/scripts/e2e/config/"
+        local_config_file_path = ".buildkite/scripts/e2e-pipeline/config/"
         config_file = "serverless_pipeline.conf" if self.project_type == "serverless" else "pipeline.conf"
         local_config_file = local_config_file_path + config_file
         container_config_file_path = "/usr/share/logstash/pipeline/logstash.conf"

.buildkite/scripts/e2e-pipeline/generate-steps.py

@@ -0,0 +1,59 @@
+import sys
+import typing
+import util
+
+from ruamel.yaml import YAML
+
+RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+TEST_COMMAND: typing.final = ".buildkite/scripts/run_e2e_tests.sh"
+
+
+def generate_test_step(stack_version, es_treeish, snapshot) -> dict:
+    label_integration_test: typing.final = f"E2E tests for {stack_version}, snapshot: {snapshot}"
+    return {
+        "label": label_integration_test,
+        "command": TEST_COMMAND,
+        "env": {
+            "SNAPSHOT": snapshot,
+            "ELASTIC_STACK_VERSION": stack_version,
+            "ELASTICSEARCH_TREEISH": es_treeish,
+            "TARGET_BRANCH": branch
+        }
+    }
+
+
+if __name__ == "__main__":
+    structure = {
+        "agents": {
+            "provider": "gcp",
+            "machineType": "n2-standard-4",
+            "imageProject": "elastic-images-prod",
+            "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204",
+            "diskSizeGb": 120
+        },
+        "steps": []}
+
+    steps = []
+    response = util.call_url_with_retry(RELEASES_URL)
+    release_json = response.json()
+    snapshots = release_json["snapshots"]
+    for snapshot_version in snapshots:
+        if snapshots[snapshot_version].startswith("7.") or snapshots[snapshot_version].startswith("8.15"):
+            continue
+        full_stack_version = snapshots[snapshot_version]
+        version_parts = snapshots[snapshot_version].split(".")
+        major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}"
+        branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions
+        steps.append(generate_test_step(full_stack_version, branch, "true"))
+
+    group_desc = f"E2E steps"
+    key_desc = "e2e-steps"
+    structure["steps"].append({
+        "group": group_desc,
+        "key": key_desc,
+        "steps": steps,
+    })
+
+    print(
+        '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
+    YAML().dump(structure, sys.stdout)

.buildkite/scripts/e2e/main.py → .buildkite/scripts/e2e-pipeline/main.py

@@ -12,10 +12,10 @@
 
 class BootstrapContextManager:
     def __enter__(self):
-        stack_version = os.environ.get("STACK_VERSION")
+        stack_version = os.environ.get("ELASTIC_STACK_VERSION")
         project_type = os.environ.get("E2E_PROJECT_TYPE", "on_prems")
         if stack_version is None:
-            raise Exception("STACK_VERSION environment variable is missing, please export and try again.")
+            raise Exception("ELASTIC_STACK_VERSION environment variable is missing, please export and try again.")
 
         print(f"Starting E2E test of Logstash running Elastic Integrations against {stack_version} version.")
         self.bootstrap = Bootstrap(stack_version, project_type)