feat(integration-tests): Modularize package tests with per-mode directories and module-scoped fixtures. #1843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

quinntaylormitchell wants to merge 12 commits into y-scope:main from quinntaylormitchell:testing-modular-redesign

+232 −147

integration-tests/.pytest.ini

-Original file line number
+Diff line change
@@ Expand Up @@
     log_cli_level = INFO
     markers =
         clp: mark tests that use the CLP storage engine
+        clp_json: mark tests that use the clp-json package
         clp_s: mark tests that use the CLP-S storage engine
+        clp_text: mark tests that use the clp-text package
+        compression: mark tests that test compression
         core: mark tests that test the CLP core binaries
-        package: mark tests that run when the CLP package is active
+        package: mark tests that use the CLP package
+        search: mark tests that test search
+        startup: mark tests that test startup

integration-tests/tests/conftest.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -7,7 +7,7 @@ @@
         "tests.fixtures.integration_test_logs",
         "tests.fixtures.path_configs",
         "tests.fixtures.package_instance",
-        "tests.fixtures.package_config",
+        "tests.fixtures.package_test_config",
     ]
@@ Expand Down @@

integration-tests/tests/fixtures/package_instance.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,35 +5,38 @@ @@
     import pytest
     from tests.utils.config import (
-        PackageConfig,
         PackageInstance,
+        PackageTestConfig,
     )
     from tests.utils.package_utils import (
         start_clp_package,
         stop_clp_package,
     )
-    @pytest.fixture
-    def fixt_package_instance(fixt_package_config: PackageConfig) -> Iterator[PackageInstance]:
+    @pytest.fixture(scope="module")
+    def fixt_package_instance(fixt_package_test_config: PackageTestConfig) -> Iterator[PackageInstance]:
         """
         Starts a CLP package instance for the given configuration and stops it during teardown.
-        :param fixt_package_config:
+        This fixture relies on `fixt_package_test_config`, and as such, the scope of this fixture should
+        never exceed that of `fixt_package_test_config`.
+        :param fixt_package_test_config:
         :return: Iterator that yields the running package instance.
         """
-        mode_name = fixt_package_config.mode_name
         try:
-            start_clp_package(fixt_package_config)
-            instance = PackageInstance(package_config=fixt_package_config)
+            start_clp_package(fixt_package_test_config)
+            instance = PackageInstance(package_test_config=fixt_package_test_config)
             yield instance
         except RuntimeError:
-            base_port = fixt_package_config.base_port
+            mode_config = fixt_package_test_config.mode_config
+            mode_name = mode_config.mode_name
+            base_port = fixt_package_test_config.base_port
             pytest.fail(
                 f"Failed to start the {mode_name} package. This could mean that one of the ports"
                 f" derived from base_port={base_port} was unavailable. You can specify a new value for"
                 " base_port with the '--base-port' flag."
             )
         finally:
-            stop_clp_package(fixt_package_config)
+            stop_clp_package(fixt_package_test_config)

...on-tests/tests/fixtures/package_config.py → ...sts/tests/fixtures/package_test_config.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -4,29 +4,25 @@
  
    import pytest

    from tests.utils.clp_mode_utils import (

        get_clp_config_from_mode,

        get_required_component_list,

    )

    from tests.utils.config import PackageConfig, PackagePathConfig

    from tests.utils.config import PackageModeConfig, PackagePathConfig, PackageTestConfig

    from tests.utils.port_utils import assign_ports_from_base

    @pytest.fixture

    def fixt_package_config(

    @pytest.fixture(scope="module")

    def fixt_package_test_config(

        request: pytest.FixtureRequest,

        fixt_package_path_config: PackagePathConfig,

    ) -> Iterator[PackageConfig]:

    ) -> Iterator[PackageTestConfig]:

        """

        Creates and maintains a PackageConfig object for a specific CLP mode.

        Creates and maintains a module-level PackageTestConfig object for a specific CLP mode. For

        efficiency, group all tests for a given mode in the same module.

        :param request:

        :return: An iterator that yields the PackageConfig object for the specified mode.

        :param request: Provides `PackageModeConfig` via `request.param`.

        :return: An iterator that yields the PackageTestConfig object for the specified mode.

        :raise ValueError: if the CLP base port's value is invalid.

        """

        mode_name: str = request.param

        clp_config_obj = get_clp_config_from_mode(mode_name)

        mode_config: PackageModeConfig = request.param

        clp_config_obj = mode_config.clp_config

        # Assign ports based on the clp base port CLI option.

        base_port_string = request.config.getoption("--base-port")

    @@ -37,18 +33,14 @@ def fixt_package_config(
  
            raise ValueError(err_msg) from err

        assign_ports_from_base(base_port, clp_config_obj)

        required_components = get_required_component_list(clp_config_obj)

        # Construct PackageConfig.

        package_config = PackageConfig(

        # Construct PackageTestConfig.

        package_test_config = PackageTestConfig(

            path_config=fixt_package_path_config,

            mode_name=mode_name,

            component_list=required_components,

            clp_config=clp_config_obj,

            mode_config=mode_config,

            base_port=base_port,

        )

        try:

            yield package_config

            yield package_test_config

        finally:

            package_config.temp_config_file_path.unlink(missing_ok=True)

            package_test_config.temp_config_file_path.unlink(missing_ok=True)

integration-tests/tests/package_tests/clp_json/__init__.py

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		"""Integration tests for the clp-json package."""

integration-tests/tests/package_tests/clp_json/test_clp_json.py

-Original file line number
+Diff line change
@@ -0,0 +1,88 @@
+    """Tests for the clp-json package."""
+    import logging
+    import pytest
+    from clp_py_utils.clp_config import (
+        ClpConfig,
+        Package,
+        QueryEngine,
+        StorageEngine,
+    )
+    from tests.utils.asserting_utils import (
+        validate_package_instance,
+    )
+    from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS
+    from tests.utils.config import PackageInstance, PackageModeConfig
+    logger = logging.getLogger(__name__)
+    # Mode description for this module.
+    CLP_JSON_MODE = PackageModeConfig(
+        mode_name="clp-json",
+        clp_config=ClpConfig(
+            package=Package(
+                storage_engine=StorageEngine.CLP_S,
+                query_engine=QueryEngine.CLP_S,
+            ),
+        ),
+        component_list=(*CLP_BASE_COMPONENTS, CLP_API_SERVER_COMPONENT),
+    )
+    # Pytest markers for this module.
+    pytestmark = [
+        pytest.mark.package,
+        pytest.mark.clp_json,
+        pytest.mark.parametrize("fixt_package_test_config", [CLP_JSON_MODE], indirect=True),
+    ]
+    @pytest.mark.startup
+    def test_clp_json_startup(fixt_package_instance: PackageInstance) -> None:
+        """
+        Validate that the `clp-json` package starts up successfully.
+        :param fixt_package_instance:
+        """
+        validate_package_instance(fixt_package_instance)
+        log_msg = "test_clp_json_startup was successful."
+        logger.info(log_msg)
+    @pytest.mark.compression
+    def test_clp_json_compression(fixt_package_instance: PackageInstance) -> None:
+        """
+        Validate that the `clp-json` package successfully compresses some dataset.
+        :param fixt_package_instance:
+        """
+        validate_package_instance(fixt_package_instance)
+        # TODO: compress some dataset and check the correctness of compression.
+        assert True
+        log_msg = "test_clp_json_compression was successful."
+        logger.info(log_msg)
+    @pytest.mark.search
+    def test_clp_json_search(fixt_package_instance: PackageInstance) -> None:
+        """
+        Validate that the `clp-json` package successfully searches some dataset.
+        :param fixt_package_instance:
+        """
+        validate_package_instance(fixt_package_instance)
+        # TODO: compress some dataset and check the correctness of compression.
+        # TODO: search through that dataset and check the correctness of the search results.
+        assert True
+        log_msg = "test_clp_json_search was successful."
+        logger.info(log_msg)

integration-tests/tests/package_tests/clp_text/__init__.py

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		"""Integration tests for the clp-text package."""

integration-tests/tests/package_tests/clp_text/test_clp_text.py

-Original file line number
+Diff line change
@@ -0,0 +1,51 @@
+    """Tests for the clp-text package."""
+    import logging
+    import pytest
+    from clp_py_utils.clp_config import (
+        ClpConfig,
+        Package,
+        QueryEngine,
+        StorageEngine,
+    )
+    from tests.utils.asserting_utils import (
+        validate_package_instance,
+    )
+    from tests.utils.clp_mode_utils import CLP_BASE_COMPONENTS
+    from tests.utils.config import PackageInstance, PackageModeConfig
+    logger = logging.getLogger(__name__)
+    # Mode description for this module.
+    CLP_TEXT_MODE = PackageModeConfig(
+        mode_name="clp-text",
+        clp_config=ClpConfig(
+            package=Package(
+                storage_engine=StorageEngine.CLP,
+                query_engine=QueryEngine.CLP,
+            ),
+            api_server=None,
+            log_ingestor=None,
+        ),
+        component_list=(*CLP_BASE_COMPONENTS,),
+    )
+    # Pytest markers for this module.
+    pytestmark = [
+        pytest.mark.package,
+        pytest.mark.clp_text,
+        pytest.mark.parametrize("fixt_package_test_config", [CLP_TEXT_MODE], indirect=True),
+    ]
+    @pytest.mark.startup
+    def test_clp_text_startup(fixt_package_instance: PackageInstance) -> None:
+        """Tests package startup."""
+        validate_package_instance(fixt_package_instance)
+        log_msg = "test_clp_text_startup was successful."
+        logger.info(log_msg)

integration-tests/tests/test_package_start.py

This file was deleted.

integration-tests/tests/utils/asserting_utils.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -37,7 +37,22 @@ def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess
  
        return proc

    def validate_package_running(package_instance: PackageInstance) -> None:

    def validate_package_instance(package_instance: PackageInstance) -> None:

        """

        Validate that the given package instance is running by performing two checks: validate that the

        instance has exactly the set of running components that it should have, and validate that the

        instance is running in the correct mode.

        :param package_instance:

        """

        # Ensure that all package components are running.

        _validate_package_running(package_instance)

        # Ensure that the package is running in the correct mode.

        _validate_running_mode_correct(package_instance)

    def _validate_package_running(package_instance: PackageInstance) -> None:

        """

        Validate that the given package instance is running by checking that the set of services running

        in the Compose project exactly matches the list of required components.

    @@ -51,7 +66,7 @@ def validate_package_running(package_instance: PackageInstance) -> None:
  
        running_services = set(list_running_services_in_compose_project(project_name))

        # Compare with list of required components.

        required_components = set(package_instance.package_config.component_list)

        required_components = set(package_instance.package_test_config.mode_config.component_list)

        if required_components == running_services:

            return

    @@ -68,7 +83,7 @@ def validate_package_running(package_instance: PackageInstance) -> None:
  
        pytest.fail(fail_msg)

    def validate_running_mode_correct(package_instance: PackageInstance) -> None:

    def _validate_running_mode_correct(package_instance: PackageInstance) -> None:

        """

        Validate that the mode described in the shared config of the instance matches the intended mode

        defined by the instance configuration. Calls pytest.fail if the shared config fails validation

    @@ -85,7 +100,7 @@ def validate_running_mode_correct(package_instance: PackageInstance) -> None:
  
        except ValidationError as err:

            pytest.fail(f"Shared config failed validation: {err}")

        intended_config = package_instance.package_config.clp_config

        intended_config = package_instance.package_test_config.mode_config.clp_config

        if not compare_mode_signatures(intended_config, running_config):

            pytest.fail("Mode mismatch: running configuration does not match intended configuration.")

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(integration-tests): Modularize package tests with per-mode directories and module-scoped fixtures. #1843

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

feat(integration-tests): Modularize package tests with per-mode directories and module-scoped fixtures. #1843

Are you sure you want to change the base?

feat(integration-tests): Modularize package tests with per-mode directories and module-scoped fixtures. #1843

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!