feat(integration-tests): Modularize package tests with per-mode directories and module-scoped fixtures.

integration-tests/.pytest.ini

@@ -17,6 +17,11 @@ log_cli_format = %(name)s %(asctime)s [%(levelname)s] %(message)s
 log_cli_level = INFO
 markers =
     clp: mark tests that use the CLP storage engine
+    clp_json: mark tests that use the clp-json package
     clp_s: mark tests that use the CLP-S storage engine
+    clp_text: mark tests that use the clp-text package
+    compression: mark tests that test compression
     core: mark tests that test the CLP core binaries
-    package: mark tests that run when the CLP package is active
+    package: mark tests that use the CLP package
+    search: mark tests that test search
+    startup: mark tests that test startup

integration-tests/tests/conftest.py

@@ -7,7 +7,7 @@
     "tests.fixtures.integration_test_logs",
     "tests.fixtures.path_configs",
     "tests.fixtures.package_instance",
-    "tests.fixtures.package_config",
+    "tests.fixtures.package_test_config",
 ]
 
 

integration-tests/tests/fixtures/package_instance.py

@@ -5,35 +5,38 @@
 import pytest
 
 from tests.utils.config import (
-    PackageConfig,
     PackageInstance,
+    PackageTestConfig,
 )
 from tests.utils.package_utils import (
     start_clp_package,
     stop_clp_package,
 )
 
 
-@pytest.fixture
-def fixt_package_instance(fixt_package_config: PackageConfig) -> Iterator[PackageInstance]:
+@pytest.fixture(scope="module")
+def fixt_package_instance(fixt_package_test_config: PackageTestConfig) -> Iterator[PackageInstance]:
     """
     Starts a CLP package instance for the given configuration and stops it during teardown.
 
-    :param fixt_package_config:
+    This fixture relies on `fixt_package_test_config`, and as such, the scope of this fixture should
+    never exceed that of `fixt_package_test_config`.
+
+    :param fixt_package_test_config:
     :return: Iterator that yields the running package instance.
     """
-    mode_name = fixt_package_config.mode_name
-
     try:
-        start_clp_package(fixt_package_config)
-        instance = PackageInstance(package_config=fixt_package_config)
+        start_clp_package(fixt_package_test_config)
+        instance = PackageInstance(package_test_config=fixt_package_test_config)
         yield instance
     except RuntimeError:
-        base_port = fixt_package_config.base_port
+        mode_config = fixt_package_test_config.mode_config
+        mode_name = mode_config.mode_name
+        base_port = fixt_package_test_config.base_port
         pytest.fail(
             f"Failed to start the {mode_name} package. This could mean that one of the ports"
             f" derived from base_port={base_port} was unavailable. You can specify a new value for"
             " base_port with the '--base-port' flag."
         )
     finally:
-        stop_clp_package(fixt_package_config)
+        stop_clp_package(fixt_package_test_config)

integration-tests/tests/fixtures/package_config.py → integration-tests/tests/fixtures/package_test_config.py

@@ -4,29 +4,25 @@
 
 import pytest
 
-from tests.utils.clp_mode_utils import (
-    get_clp_config_from_mode,
-    get_required_component_list,
-)
-from tests.utils.config import PackageConfig, PackagePathConfig
+from tests.utils.config import PackageModeConfig, PackagePathConfig, PackageTestConfig
 from tests.utils.port_utils import assign_ports_from_base
 
 
-@pytest.fixture
-def fixt_package_config(
+@pytest.fixture(scope="module")
+def fixt_package_test_config(
     request: pytest.FixtureRequest,
     fixt_package_path_config: PackagePathConfig,
-) -> Iterator[PackageConfig]:
+) -> Iterator[PackageTestConfig]:
     """
-    Creates and maintains a PackageConfig object for a specific CLP mode.
+    Creates and maintains a module-level PackageTestConfig object for a specific CLP mode. For
+    efficiency, group all tests for a given mode in the same module.
 
-    :param request:
-    :return: An iterator that yields the PackageConfig object for the specified mode.
+    :param request: Provides `PackageModeConfig` via `request.param`.
+    :return: An iterator that yields the PackageTestConfig object for the specified mode.
     :raise ValueError: if the CLP base port's value is invalid.
     """
-    mode_name: str = request.param
-
-    clp_config_obj = get_clp_config_from_mode(mode_name)
+    mode_config: PackageModeConfig = request.param
+    clp_config_obj = mode_config.clp_config
 
     # Assign ports based on the clp base port CLI option.
     base_port_string = request.config.getoption("--base-port")
@@ -37,18 +33,14 @@ def fixt_package_config(
         raise ValueError(err_msg) from err
     assign_ports_from_base(base_port, clp_config_obj)
 
-    required_components = get_required_component_list(clp_config_obj)
-
-    # Construct PackageConfig.
-    package_config = PackageConfig(
+    # Construct PackageTestConfig.
+    package_test_config = PackageTestConfig(
         path_config=fixt_package_path_config,
-        mode_name=mode_name,
-        component_list=required_components,
-        clp_config=clp_config_obj,
+        mode_config=mode_config,
         base_port=base_port,
     )
 
     try:
-        yield package_config
+        yield package_test_config
     finally:
-        package_config.temp_config_file_path.unlink(missing_ok=True)
+        package_test_config.temp_config_file_path.unlink(missing_ok=True)

integration-tests/tests/package_tests/clp_json/__init__.py

@@ -0,0 +1 @@
+"""Integration tests for the clp-json package."""

integration-tests/tests/package_tests/clp_json/test_clp_json.py

@@ -0,0 +1,88 @@
+"""Tests for the clp-json package."""
+
+import logging
+
+import pytest
+from clp_py_utils.clp_config import (
+    ClpConfig,
+    Package,
+    QueryEngine,
+    StorageEngine,
+)
+
+from tests.utils.asserting_utils import (
+    validate_package_instance,
+)
+from tests.utils.clp_mode_utils import CLP_API_SERVER_COMPONENT, CLP_BASE_COMPONENTS
+from tests.utils.config import PackageInstance, PackageModeConfig
+
+logger = logging.getLogger(__name__)
+
+
+# Mode description for this module.
+CLP_JSON_MODE = PackageModeConfig(
+    mode_name="clp-json",
+    clp_config=ClpConfig(
+        package=Package(
+            storage_engine=StorageEngine.CLP_S,
+            query_engine=QueryEngine.CLP_S,
+        ),
+    ),
+    component_list=(*CLP_BASE_COMPONENTS, CLP_API_SERVER_COMPONENT),
+)
+
+
+# Pytest markers for this module.
+pytestmark = [
+    pytest.mark.package,
+    pytest.mark.clp_json,
+    pytest.mark.parametrize("fixt_package_test_config", [CLP_JSON_MODE], indirect=True),
+]
+
+
+@pytest.mark.startup
+def test_clp_json_startup(fixt_package_instance: PackageInstance) -> None:
+    """
+    Validate that the `clp-json` package starts up successfully.
+
+    :param fixt_package_instance:
+    """
+    validate_package_instance(fixt_package_instance)
+
+    log_msg = "test_clp_json_startup was successful."
+    logger.info(log_msg)
+
+
+@pytest.mark.compression
+def test_clp_json_compression(fixt_package_instance: PackageInstance) -> None:
+    """
+    Validate that the `clp-json` package successfully compresses some dataset.
+
+    :param fixt_package_instance:
+    """
+    validate_package_instance(fixt_package_instance)
+
+    # TODO: compress some dataset and check the correctness of compression.
+    assert True
+
+    log_msg = "test_clp_json_compression was successful."
+    logger.info(log_msg)
+
+
+@pytest.mark.search
+def test_clp_json_search(fixt_package_instance: PackageInstance) -> None:
+    """
+    Validate that the `clp-json` package successfully searches some dataset.
+
+    :param fixt_package_instance:
+    """
+    validate_package_instance(fixt_package_instance)
+
+    # TODO: compress some dataset and check the correctness of compression.
+
+    # TODO: search through that dataset and check the correctness of the search results.
+
+    assert True
+
+    log_msg = "test_clp_json_search was successful."
+    logger.info(log_msg)

integration-tests/tests/package_tests/clp_text/__init__.py

@@ -0,0 +1 @@
+"""Integration tests for the clp-text package."""

integration-tests/tests/package_tests/clp_text/test_clp_text.py

@@ -0,0 +1,51 @@
+"""Tests for the clp-text package."""
+
+import logging
+
+import pytest
+from clp_py_utils.clp_config import (
+    ClpConfig,
+    Package,
+    QueryEngine,
+    StorageEngine,
+)
+
+from tests.utils.asserting_utils import (
+    validate_package_instance,
+)
+from tests.utils.clp_mode_utils import CLP_BASE_COMPONENTS
+from tests.utils.config import PackageInstance, PackageModeConfig
+
+logger = logging.getLogger(__name__)
+
+
+# Mode description for this module.
+CLP_TEXT_MODE = PackageModeConfig(
+    mode_name="clp-text",
+    clp_config=ClpConfig(
+        package=Package(
+            storage_engine=StorageEngine.CLP,
+            query_engine=QueryEngine.CLP,
+        ),
+        api_server=None,
+        log_ingestor=None,
+    ),
+    component_list=(*CLP_BASE_COMPONENTS,),
+)
+
+
+# Pytest markers for this module.
+pytestmark = [
+    pytest.mark.package,
+    pytest.mark.clp_text,
+    pytest.mark.parametrize("fixt_package_test_config", [CLP_TEXT_MODE], indirect=True),
+]
+
+
+@pytest.mark.startup
+def test_clp_text_startup(fixt_package_instance: PackageInstance) -> None:
+    """Tests package startup."""
+    validate_package_instance(fixt_package_instance)
+
+    log_msg = "test_clp_text_startup was successful."
+    logger.info(log_msg)

integration-tests/tests/utils/asserting_utils.py

@@ -37,7 +37,22 @@ def run_and_assert(cmd: list[str], **kwargs: Any) -> subprocess.CompletedProcess
     return proc
 
 
-def validate_package_running(package_instance: PackageInstance) -> None:
+def validate_package_instance(package_instance: PackageInstance) -> None:
+    """
+    Validate that the given package instance is running by performing two checks: validate that the
+    instance has exactly the set of running components that it should have, and validate that the
+    instance is running in the correct mode.
+
+    :param package_instance:
+    """
+    # Ensure that all package components are running.
+    _validate_package_running(package_instance)
+
+    # Ensure that the package is running in the correct mode.
+    _validate_running_mode_correct(package_instance)
+
+
+def _validate_package_running(package_instance: PackageInstance) -> None:
     """
     Validate that the given package instance is running by checking that the set of services running
     in the Compose project exactly matches the list of required components.
@@ -51,7 +66,7 @@ def validate_package_running(package_instance: PackageInstance) -> None:
     running_services = set(list_running_services_in_compose_project(project_name))
 
     # Compare with list of required components.
-    required_components = set(package_instance.package_config.component_list)
+    required_components = set(package_instance.package_test_config.mode_config.component_list)
     if required_components == running_services:
         return
 
@@ -68,7 +83,7 @@ def validate_package_running(package_instance: PackageInstance) -> None:
     pytest.fail(fail_msg)
 
 
-def validate_running_mode_correct(package_instance: PackageInstance) -> None:
+def _validate_running_mode_correct(package_instance: PackageInstance) -> None:
     """
     Validate that the mode described in the shared config of the instance matches the intended mode
     defined by the instance configuration. Calls pytest.fail if the shared config fails validation
@@ -85,7 +100,7 @@ def validate_running_mode_correct(package_instance: PackageInstance) -> None:
     except ValidationError as err:
         pytest.fail(f"Shared config failed validation: {err}")
 
-    intended_config = package_instance.package_config.clp_config
+    intended_config = package_instance.package_test_config.mode_config.clp_config
 
     if not compare_mode_signatures(intended_config, running_config):
         pytest.fail("Mode mismatch: running configuration does not match intended configuration.")