Display container logs, increase VM spec.

Author: mashhurs

.buildkite/scripts/e2e-pipeline/generate-steps.py

@@ -59,7 +59,7 @@ def generate_steps_for_main_branch(versions) -> list:
     structure = {
         "agents": {
             "provider": "gcp",
-            "machineType": "n2-standard-4",
+            "machineType": "n2-standard-16",
             "imageProject": "elastic-images-prod",
             "image": "family/platform-ingest-logstash-multi-jdk-ubuntu-2204",
             "diskSizeGb": 120

.buildkite/scripts/e2e-pipeline/main.py

@@ -50,13 +50,9 @@ def main(skip_setup=False, integrations=[]):
                 print(f"Test failed for {package} with {e}.")
                 failed_packages.append(package)
 
-        container = util.get_logstash_container()
-
-        # pretty printing
-        print(f"Logstash docker container logs..")
-        ls_container_logs = container.logs().decode('utf-8')
-        for log_line in ls_container_logs.splitlines():
-            print(log_line)
+    util.show_container_logs("elastic-package-stack-e2e-logstash-1")
+    util.show_container_logs("elastic-package-stack-e2e-elasticsearch-1")
+    util.show_container_logs("elastic-package-stack-e2e-elastic-agent-1")
 
     if len(failed_packages) > 0:
         raise Exception(f"Following packages failed: {failed_packages}")

.buildkite/scripts/e2e-pipeline/util.py

@@ -18,11 +18,15 @@ def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> reque
     session.mount(schema, HTTPAdapter(max_retries=retries))
     return session.get(url)
 
-
-def get_logstash_container() -> Container:
+def show_container_logs(container_name: str):
     client = docker.from_env()
-    return client.containers.get("elastic-package-stack-e2e-logstash-1")
+    container = client.containers.get(container_name)
 
+    # pretty printing
+    print(f"{container_name} docker container logs..")
+    container_logs = container.logs().decode('utf-8')
+    for log_line in container_logs.splitlines():
+        print(log_line)
 
 def run_or_raise_error(commands: list, error_message):
     result = subprocess.run(commands, universal_newlines=True, stdout=subprocess.PIPE)