Greater than 9.3 stack versions defined elastic-package env uses elastic_agent without setting SSL. With this change, Logstash is also opting out SSL settings for the versions greater than 9.3

Author: mashhurs

.buildkite/scripts/e2e-pipeline/bootstrap.py

@@ -38,6 +38,7 @@ def __init__(self, stack_version: str, project_type: str) -> None:
         Returns:
             Validates and sets stack version, project type and resolves elastic package distro based on running OS (sys.platform)
         """
+        print(f"Stack version: {stack_version}")
         self.stack_version = stack_version
         self.__validate_and_set_project_type(project_type)
         self.__resolve_distro()
@@ -160,9 +161,33 @@ def __reload_container(self) -> None:
                                 "Error occurred while reloading Logstash container, see logs for details.")
         time.sleep(20)  # give a time Logstash pipeline to fully start
 
+    def __is_version_gte(self, version: str, target_major: int, target_minor: int) -> bool:
+        # Remove -SNAPSHOT suffix if present
+        clean_version = version.replace("-SNAPSHOT", "")
+        parts = clean_version.split(".")
+        if len(parts) < 2:
+            return False
+        
+        try:
+            major = int(parts[0])
+            minor = int(parts[1])
+            if major > target_major:
+                return True
+            elif major == target_major:
+                return minor >= target_minor
+            else:
+                return False
+        except ValueError:
+            return False
+
     def __update_pipeline_config(self) -> None:
         local_config_file_path = ".buildkite/scripts/e2e-pipeline/config/"
-        config_file = "serverless_pipeline.conf" if self.project_type == "serverless" else "pipeline.conf"
+        if self.__is_version_gte(self.stack_version, 9, 3):
+            config_file = "pipeline-ea-without-ssl.conf"
+        elif self.project_type == "serverless":
+            config_file = "serverless_pipeline.conf"
+        else:
+            config_file = "pipeline.conf"
         local_config_file = local_config_file_path + config_file
         container_config_file_path = "/usr/share/logstash/pipeline/logstash.conf"
         # python docker client (internally uses subprocess) requires special TAR header with tar operations

.buildkite/scripts/e2e-pipeline/config/pipeline-ea-without-ssl.conf

@@ -0,0 +1,28 @@
+input {
+  elastic_agent {
+    port => 5044
+  }
+}
+
+
+filter {
+  elastic_integration {
+    remove_field => ['@version']
+    hosts => ["https://elasticsearch:9200"]
+    username => "${ELASTIC_USER}"
+    password => "${ELASTIC_PASSWORD}"
+    ssl_enabled => true
+    ssl_verification_mode => "none"
+  }
+}
+
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "${ELASTIC_USER}"
+    password => "${ELASTIC_PASSWORD}"
+    ssl_enabled => true
+    ssl_verification_mode => "none"
+  }
+}

.buildkite/scripts/e2e-pipeline/main.py

@@ -42,8 +42,6 @@ def main(skip_setup=False, integrations=[]):
         working_dir = os.getcwd()
         test_plugin = PluginTest()
 
-        util.show_containers_logs(["logstash-", "elasticsearch-", "elastic-agent-"])
-
         packages = integrations or INTEGRATION_PACKAGES_TO_TEST
         for package in packages:
             try:

.buildkite/scripts/e2e-pipeline/plugin_test.py

@@ -58,5 +58,5 @@ def on(self, package: str) -> None:
                 for result_line in result.stdout.splitlines(): print(f"{result_line}")
 
         # although there was an error, le's check how LS performed and make sure errors weren't because of Logstash
-        time.sleep(20)
+        time.sleep(2) # make sure LS processes the event way to downstream ES
         self.__analyze_logstash_throughput(package, result)