[main branch] Rebuilds pull-request, build and E2E pipelines.

.buildkite/build-pipeline.yml

@@ -1,43 +1,28 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+- label: "Build pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
 
-agents:
-  provider: "gcp"
-  machineType: "n1-standard-4"
-  image: family/core-ubuntu-2204
-
-steps:
-  - label: ":hammer: Build plugin with LS & ES 8.x :elasticsearch:"
-    # Builds with LS and ES last 8.x released version
-    # Runs integration tests on 8.x released versions
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      SNAPSHOT: false
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-      TARGET_BRANCH: "8.x"
-
-  - label: ":hammer: Build plugin with LS 8.x-SNAPSHOT & ES `main` branch :elasticsearch:"
-    # Builds with LS last 8.x released version and ES main
-    # Runs integration tests on 8.x released versions
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      ELASTICSEARCH_TREEISH: "main"
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-
-  - label: ":hammer: Build plugin with LS & ES `main` branch :elasticsearch:"
-    # Builds with LS last 8.x released version and ES main
-    # Runs integration tests on 8.x released versions
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "main"
-      ELASTICSEARCH_TREEISH: "main"
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+  
+    echo "--- Generating dynamic steps"
+    set +e
+    python3 .buildkite/scripts/build-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+  
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/e2e-pipeline.yml

@@ -1,39 +1,28 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+- label: "E2E pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
 
-agents:
-  provider: gcp
-  imageProject: elastic-images-prod
-  image: family/platform-ingest-logstash-multi-jdk-ubuntu-2204
-  machineType: "n2-standard-4"
-  diskSizeGb: 120
-
-steps:
-  # ------------- Run E2E tests ---------------------
-  - label: ":test_tube: Run E2E tests with LS 8.x :rocket:"
-    # uses the LS main & plugin 8.x branch when building the plugin
-    # Runs integration tests against 8.x release version
-    command:
-      - .buildkite/scripts/run_e2e_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      TARGET_BRANCH: "8.x"
-
-  - label: ":test_tube: Run E2E tests with LS 8.x-SNAPSHOT :rocket:"
-    # uses the LS & plugin main branch when building the plugin
-    # Runs integration tests against 8.x-SNAPSHOT version
-    command:
-      - .buildkite/scripts/run_e2e_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: "8.current"
-      SNAPSHOT: true
-      TARGET_BRANCH: "8.x"
-
-  - label: ":test_tube: Run E2E tests with LS `main` :rocket:"
-    # uses the LS & plugin main branch when building the plugin
-    # Runs integration tests against snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json
-    command:
-      - .buildkite/scripts/run_e2e_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: main
-      SNAPSHOT: true
-      ELASTICSEARCH_TREEISH: main
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
+
+    echo "--- Generating steps dynamically"
+    set +e
+    python3 .buildkite/scripts/e2e-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/pull-request-pipeline.yml

@@ -1,32 +1,29 @@
-# define a GCP VM agent to support container management (by default agent doesn't support)
-agents:
-  provider: "gcp"
-  machineType: "n1-standard-4"
-  image: family/core-ubuntu-2204
+- label: "Pull request pipeline"
+  command: |
+    #!/usr/bin/env bash
+    set -eo pipefail
+    echo "--- Downloading prerequisites"
+    python3 -m pip install ruamel.yaml
+    python3 -m pip install requests
+    curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+    chmod a+x /usr/bin/yq
 
-steps:
-  # ------------- Unit tests ---------------------
-  - label: ":hammer: Unit tests with LS & ES main :docker:"
-    # Builds the plugin (with current changes) against LS and ES main
-    # Runs unit tests on LS & ES main docker
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: main
-      ELASTICSEARCH_TREEISH: main
-      INTEGRATION: false
-      SNAPSHOT: true
+    export TARGET_BRANCH="$GITHUB_PR_TARGET_BRANCH"
 
-  # ------------- Integration tests ---------------------
-  - label: ":hammer: Integration tests with LS & ES main :docker:"
-    # Builds the plugin (with current changes) against LS and ES main
-    # Runs integration tests on snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json
-    command:
-      - .buildkite/scripts/run_tests.sh
-    env:
-      ELASTIC_STACK_VERSION: main
-      ELASTICSEARCH_TREEISH: main
-      SNAPSHOT: true
-      INTEGRATION: true
-      SECURE_INTEGRATION: true
-      LOG_LEVEL: "info"
+    echo "--- Generating steps dynamically"
+    set +e
+    python3 .buildkite/scripts/pull-request-pipeline/generate-steps.py > pipeline_steps.yml
+    if [[ $$? -ne 0 ]]; then
+      echo "^^^ +++"
+      echo "There was a problem with generating pipeline steps."
+      cat pipeline_steps.yml
+      echo "Exiting now."
+      exit 1
+    else
+      set -eo pipefail
+      cat pipeline_steps.yml | yq .
+    fi
+
+    set -eo pipefail
+    echo "--- Uploading steps to buildkite"
+    cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/scripts/build-pipeline/generate-steps.py

@@ -0,0 +1,95 @@
+import os
+import requests
+import sys
+import typing
+from requests.adapters import HTTPAdapter, Retry
+
+from ruamel.yaml import YAML
+
+RELEASES_URL = "https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+TEST_COMMAND: typing.final = ".buildkite/scripts/run_tests.sh"
+
+
+def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response:
+    schema = "https://" if "https://" in url else "http://"
+    session = requests.Session()
+    # retry on most common failures such as connection timeout(408), etc...
+    retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504])
+    session.mount(schema, HTTPAdapter(max_retries=retries))
+    return session.get(url)
+
+
+def generate_test_step(stack_version, branch, snapshot) -> dict:
+    label_integration_test: typing.final = f"Integration test for {stack_version}, snapshot: {snapshot}"
+    step: dict = {
+        "label": label_integration_test,
+        "command": TEST_COMMAND,
+        "env": {
+            "SNAPSHOT": snapshot,
+            "ELASTIC_STACK_VERSION": stack_version,
+            "INTEGRATION": "true",
+            "SECURE_INTEGRATION": "true",
+            "LOG_LEVEL": "info"
+        }
+    }
+    # we are not going to set branch if job kicked of through webhook (PR merge or manual PR run)
+    if branch is not None:
+        step["env"]["TARGET_BRANCH"] = branch
+    return step
+
+
+def generate_steps_for_scheduler(versions) -> list:
+    steps: list = []
+    snapshots = versions["snapshots"]
+    for snapshot_version in snapshots:
+        if snapshots[snapshot_version].startswith("7."):
+            continue
+        full_stack_version = snapshots[snapshot_version]
+        version_parts = snapshots[snapshot_version].split(".")
+        major_minor_versions = snapshot_version if snapshot_version == "main" else f"{version_parts[0]}.{version_parts[1]}"
+        branch = f"{version_parts[0]}.x" if snapshot_version.find("future") > -1 else major_minor_versions
+        steps.append(generate_test_step(full_stack_version, branch, "true"))
+    return steps
+
+
+def generate_steps_for_main_branch(versions) -> list:
+    steps: list = []
+    full_stack_version: typing.final = versions["snapshots"]["main"]
+    steps.append(generate_test_step(full_stack_version, None, "true"))
+    return steps
+
+
+if __name__ == "__main__":
+    structure = {
+        "agents": {
+            "provider": "gcp",
+            "machineType": "n1-standard-4",
+            "image": "family/core-ubuntu-2204"
+        },
+        "steps": []}
+
+    response = call_url_with_retry(RELEASES_URL)
+    versions_json: typing.final = response.json()
+
+    # Use BUILDKITE_SOURCE to figure out PR merge or schedule.
+    # If PR merge, no need to run builds on all branches, target branch will be good
+    #   - webhook when PR gets merged
+    #   - schedule when daily schedule starts
+    #   - ui when manually kicking job from BK UI
+    #       - manual kick off will be on PR or entire main branch, can be decided with BUILDKITE_BRANCH
+    bk_source = os.getenv("BUILDKITE_SOURCE")
+    bk_branch = os.getenv("BUILDKITE_BRANCH")
+    steps = generate_steps_for_scheduler(versions_json) if (bk_source == "schedule" or bk_branch == "main") \
+        else generate_steps_for_main_branch(versions_json)
+
+    group_desc = f"Build steps"
+    key_desc = "build-steps"
+    structure["steps"].append({
+        "group": group_desc,
+        "key": key_desc,
+        "steps": steps,
+    })
+
+    print(
+        '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
+    YAML().dump(structure, sys.stdout)

.buildkite/scripts/e2e/README.md → .buildkite/scripts/e2e-pipeline/README.md

@@ -32,19 +32,19 @@ In order to run tests with serverless, you also need to export `EC_API_KEY` whic
 In the pipelines, this will be automatically retrieved from Vault services.
 
 #### Stack version
-E2E also requires `STACK_VERSION` (ex: "8.12.0") environment variable in order to test against.
+E2E also requires `ELASTIC_STACK_VERSION` (ex: "8.12.0") environment variable in order to test against.
 Make sure to export it before running. In the Buildkite pipeline, this var will be resolved and exported. 
 
 #### Installing dependencies
 Make sure you have python installed on you local
 ```bash
-pip install -r .buildkite/scripts/e2e/requirements.txt
+pip install -r .buildkite/scripts/e2e-pipeline/requirements.txt
 ```
 
 ### Run
 Run the following command from the repo dir:
 ```bash
-python3 .buildkite/scripts/e2e/main.py
+python3 .buildkite/scripts/e2e-pipeline/main.py
 ```
 
 ## Troubleshooting

.buildkite/scripts/e2e/bootstrap.py → .buildkite/scripts/e2e-pipeline/bootstrap.py

@@ -122,7 +122,7 @@ def __reload_container(self) -> None:
         time.sleep(20)  # give a time Logstash pipeline to fully start
 
     def __update_pipeline_config(self) -> None:
-        local_config_file_path = ".buildkite/scripts/e2e/config/"
+        local_config_file_path = ".buildkite/scripts/e2e-pipeline/config/"
         config_file = "serverless_pipeline.conf" if self.project_type == "serverless" else "pipeline.conf"
         local_config_file = local_config_file_path + config_file
         container_config_file_path = "/usr/share/logstash/pipeline/logstash.conf"