diff --git a/.buildkite/scripts/e2e-pipeline/README.md b/.buildkite/scripts/e2e-pipeline/README.md index f62be8fd..9857ec44 100644 --- a/.buildkite/scripts/e2e-pipeline/README.md +++ b/.buildkite/scripts/e2e-pipeline/README.md @@ -41,12 +41,15 @@ Make sure you have python installed on you local pip install -r .buildkite/scripts/e2e-pipeline/requirements.txt ``` -### Run +### Run on local Run the following command from the repo dir: ```bash -python3 .buildkite/scripts/e2e-pipeline/main.py +python3 .buildkite/scripts/e2e-pipeline/main.py --skip-setup=true --integrations='apache','nginx' ``` +This will run entire ELK docker containers. +Remove `--skip-setup` or use `--skip-setup=true` if you are running the script for the first time, where it needs to set up elastic-package and integrations. + ## Troubleshooting - The project retries on some operations to overcome timeout issues, uses [`retry` tool](https://formulae.brew.sh/formula/retry). If you get `retry` undefined error, make sure to install it. ``` diff --git a/.buildkite/scripts/e2e-pipeline/bootstrap.py b/.buildkite/scripts/e2e-pipeline/bootstrap.py index 13d39f81..2a369b9c 100644 --- a/.buildkite/scripts/e2e-pipeline/bootstrap.py +++ b/.buildkite/scripts/e2e-pipeline/bootstrap.py @@ -14,6 +14,7 @@ YAML = ruamel.yaml.YAML() + class Bootstrap: ELASTIC_PACKAGE_DISTRO_URL = "https://api.github.com/repos/elastic/elastic-package/releases/latest" LOGSTASH_CONTAINER_NAME = "elastic-package-stack-e2e-logstash-1" @@ -87,7 +88,7 @@ def __scan_for_unsupported_processors(self) -> None: curr_dir = os.getcwd() pipeline_definition_file_path = "integrations/packages/**/data_stream/**/elasticsearch/ingest_pipeline/*.yml" files = glob.glob(os.path.join(curr_dir, pipeline_definition_file_path)) - unsupported_processors: dict[list] = {} # {processor_type: list} + unsupported_processors: dict[list] = {} # {processor_type: list} for file in files: try: @@ -185,15 +186,16 @@ def __teardown_stack(self) -> None: util.run_or_raise_error(["elastic-package", "stack", "down"], "Error occurred while stopping stacks with elastic-package. Check logs for details.") - def run_elastic_stack(self) -> None: + def run_elastic_stack(self, skip_setup=False) -> None: """ Downloads elastic-package, creates a profile and runs ELK, Fleet, ERP and elastic-agent """ - self.__download_elastic_package() - self.__make_elastic_package_global() - self.__clone_integrations_repo() - self.__scan_for_unsupported_processors() - self.__setup_elastic_package_profile() + if not skip_setup: + self.__download_elastic_package() + self.__make_elastic_package_global() + self.__clone_integrations_repo() + self.__scan_for_unsupported_processors() + self.__setup_elastic_package_profile() self.__spin_stack() self.__install_plugin() self.__reload_container() diff --git a/.buildkite/scripts/e2e-pipeline/main.py b/.buildkite/scripts/e2e-pipeline/main.py index de6d5ca6..cd6321c4 100644 --- a/.buildkite/scripts/e2e-pipeline/main.py +++ b/.buildkite/scripts/e2e-pipeline/main.py @@ -2,6 +2,7 @@ Main entry point of the E2E test suites """ +import argparse import os from bootstrap import Bootstrap from plugin_test import PluginTest @@ -11,6 +12,10 @@ class BootstrapContextManager: + def __init__(self, skip_setup=False): + # save args as attributes + self.skip_setup = skip_setup + def __enter__(self): stack_version = os.environ.get("ELASTIC_STACK_VERSION") project_type = os.environ.get("E2E_PROJECT_TYPE", "on_prems") @@ -19,7 +24,7 @@ def __enter__(self): print(f"Starting E2E test of Logstash running Elastic Integrations against {stack_version} version.") self.bootstrap = Bootstrap(stack_version, project_type) - self.bootstrap.run_elastic_stack() + self.bootstrap.run_elastic_stack(self.skip_setup) return self.bootstrap def __exit__(self, exc_type, exc_value, traceback): @@ -30,13 +35,14 @@ def __exit__(self, exc_type, exc_value, traceback): self.bootstrap.stop_elastic_stack() -def main(): +def main(skip_setup=False, integrations=[]): failed_packages = [] - with BootstrapContextManager() as bootstrap: + with BootstrapContextManager(skip_setup) as bootstrap: working_dir = os.getcwd() test_plugin = PluginTest() - for package in INTEGRATION_PACKAGES_TO_TEST: + packages = integrations or INTEGRATION_PACKAGES_TO_TEST + for package in packages: try: os.chdir(f"{working_dir}/integrations/packages/{package}") test_plugin.on(package) @@ -57,4 +63,13 @@ def main(): if __name__ == "__main__": - main() + parser = argparse.ArgumentParser() + parser.add_argument('--skip-setup, 'action='store_false') + parser.add_argument('--integrations') + args = parser.parse_args() + + skip_setup = args.skip_setup + integrations = args.integrations.split(',') if args.integrations else [] + + print(f"Running with --skip-setup:{skip_setup}, --integrations:{integrations}") + main(skip_setup, integrations)